diff options
| author | Harald Musum <musum@verizonmedia.com> | 2020-10-12 07:43:17 +0200 |
|---|---|---|
| committer | Harald Musum <musum@verizonmedia.com> | 2020-10-12 07:43:17 +0200 |
| commit | 99c036b9b5acafa0c3ea667132aaef4939c6fcf9 (patch) | |
| tree | 7bd08ac6ec1a0f2c5aa7339867cc649bbce88ba9 | |
| parent | f8cbda74df298dbe1c18dea3537251b5c0049a3a (diff) | |
| parent | c3f03aa13bfa7e34ae342df51fcfa043d11ea4d3 (diff) | |
Merge branch 'master' into hmusum/remove-unused-flag-7
642 files changed, 5319 insertions, 6391 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 1a05ffe5073..b4a19b2d684 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -53,7 +53,7 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST endif() endif() else() - set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor -Wformat-security") + set(CXX_SPECIFIC_WARN_OPTS "-Wnoexcept -Wsuggest-override -Wnon-virtual-dtor -Wformat-security") if(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8" OR (VESPA_OS_DISTRO STREQUAL "rhel" AND VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "8" AND diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index 0d4c0ad76eb..8415781b827 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -11,7 +11,6 @@ import com.yahoo.config.FileReference; */ public interface FileRegistry { - FileReference addFile(String relativePath); FileReference addUri(String uri); default FileReference addApplicationPackage() { return addFile(""); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java index 2a798839752..ffe011af1e8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.provision.ClusterSpec; + import java.util.List; +import java.util.Optional; /** * Contains the action to be performed on the given services to handle a config change @@ -38,6 +41,10 @@ public interface ConfigChangeAction { /** Returns whether this change should be allowed */ boolean allowed(); + /** The id of the cluster that needs this action applied */ + // TODO: Remove this default implementation after October 2020 + default ClusterSpec.Id clusterId() { return null; } + /** Returns whether this change should be ignored for internal redeploy */ default boolean ignoreForInternalRedeploy() { return false; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ServiceInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ServiceInfo.java index bbf14d7dcea..7899a6ff5b0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ServiceInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ServiceInfo.java @@ -85,4 +85,9 @@ public class ServiceInfo { return result; } + @Override + public String toString() { + return "service '" + serviceName + "' of type " + serviceType + " on " + hostName; + } + } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 10649df88e1..2594c64b951 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -140,7 +140,7 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> getFiles(Path dir,String fileSuffix,boolean recurse) { + public List<NamedReader> getFiles(Path dir, String fileSuffix, boolean recurse) { return new ArrayList<>(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java index e3bbf78ec21..6189276707b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java @@ -8,6 +8,7 @@ import com.yahoo.searchdefinition.derived.DerivedConfiguration; * @author mathiasm */ public abstract class Validator { + protected DerivedConfiguration config; protected Search search; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 5ef01f986c0..e75a9f2b125 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -56,7 +56,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon private String preload = Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"); // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump. - private long mmapNoCoreLimit = -1l; + private long mmapNoCoreLimit = -1L; // If this is true it will dump core when OOM private boolean coreOnOOM = false; @@ -77,7 +77,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * more key/value pairs to the service-list dump. * Supported key datatypes are String, and values may be String or Integer. */ - private Map<String, Object> serviceProperties = new LinkedHashMap<>(); + private final Map<String, Object> serviceProperties = new LinkedHashMap<>(); /** The affinity properties of this service. */ private Optional<Affinity> affinity = Optional.empty(); @@ -176,15 +176,6 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon } else { // User defined from spec wantedPort = userWantedPort; -/* if ((wantedPort >= Host.BASE_PORT) && - (wantedPort <= (Host.BASE_PORT + Host.MAX_PORTS))) { - throw new RuntimeException - ("Attribute 'basePort=" + wantedPort + - "' is not allowed to be inside Vespa's reserved port range " - + Host.BASE_PORT + "-" - + (Host.BASE_PORT + Host.MAX_PORTS) + "."); - } -*/ } return wantedPort; } @@ -210,20 +201,18 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * Computes and returns the i'th port for this service, based on * this Service's baseport. * - * @param i The offset from 'basePort' of the port to return + * @param i the offset from 'basePort' of the port to return * @return the i'th port relative to the base port. * @throws IllegalStateException if i is out of range. */ public int getRelativePort(int i) { if (ports.size() < 1) { - throw new IllegalStateException - ("Requested port with offset " + i + " for service that " + - "has not reserved any ports: " + this); + throw new IllegalStateException("Requested port with offset " + i + " for service that " + + "has not reserved any ports: " + this); } if (i >= ports.size()) { - throw new IllegalStateException - ("Requested port with offset " + i + " for service that " + - "only has reserved " + ports.size() + " ports: " + this); + throw new IllegalStateException("Requested port with offset " + i + " for service that " + + "only has reserved " + ports.size() + " ports: " + this); } return ports.get(i); } @@ -269,20 +258,10 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon return myClass.getName().substring(1 + myPackage.getName().length()); } - /** - * @return the physical host on which this service runs. - */ - public Host getHost() { - if (hostResource != null) { - return hostResource.getHost(); - } else { - return null; - } - } + @Override + public HostResource getHost() { return hostResource; } - /** - * @return The hostname on which this service runs. - */ + @Override public String getHostName() { return hostResource.getHostname(); } @@ -306,7 +285,6 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon protected int getIndex(HostResource host) { int i = 0; for (Service s : host.getServices()) { - //if (s.getClass().equals(getClass()) && (s != this)) { if (s.getServiceType().equals(getServiceType()) && (s != this)) { i++; } @@ -448,7 +426,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon } /** - * WARNING: should only be called before initService(), otherwise call at own risk! + * WARNING: should only be called before initService() */ public void setBasePort(int wantedPort) { this.basePort = wantedPort; @@ -465,27 +443,27 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon /** * Add the given file to the application's file distributor. * - * @param relativePath path to the file, relative to the app package. + * @param relativePath path to the file, relative to the app package * @return the file reference hash */ public FileReference sendFile(String relativePath) { - return getRoot().getFileDistributor().sendFileToHost(relativePath, getHost()); + Host host = null; + if (getHost() != null) // false when running application tests without hosts + host = getHost().getHost(); + return getRoot().getFileDistributor().sendFileToHost(relativePath, host); } public FileReference sendUri(String uri) { - return getRoot().getFileDistributor().sendUriToHost(uri, getHost()); + return getRoot().getFileDistributor().sendUriToHost(uri, getHost().getHost()); } - /** - * - * The service HTTP port for health status - * @return portnumber - */ + /** The service HTTP port for health status */ public int getHealthPort() { return -1;} /** * Overridden by subclasses. List of default dimensions to be added to this services metrics - * @return The default dimensions for this service + * + * @return the default dimensions for this service */ public HashMap<String, String> getDefaultMetricDimensions(){ return new LinkedHashMap<>(); } @@ -510,4 +488,5 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon public String toString() { return getServiceName() + " on " + (getHost() == null ? "no host" : getHost().toString()); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 1c73000cc24..27fb0f444ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -47,7 +47,7 @@ public class HostResource implements Comparable<HostResource> { } /** - * Return the currently bounded {@link com.yahoo.vespa.model.Host}. + * Return the currently bound {@link com.yahoo.vespa.model.Host}. * * @return the {@link com.yahoo.vespa.model.Host} if bound, null if not. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/Service.java index 25128c5ab14..b102ccebee0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Service.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Service.java @@ -29,16 +29,11 @@ public interface Service extends ConfigProducer, NetworkPortRequestor { */ Optional<String> getPreShutdownCommand(); - /** - * Returns a PortsMeta object, giving access to more information - * about the different ports of this service. - */ + /** Returns a PortsMeta object, giving access to more information about the different ports of this service. */ PortsMeta getPortsMeta(); - /** - * @return the physical host on which this service runs. - */ - Host getHost(); + /** Returns the physical host resource on which this service runs. */ + HostResource getHost(); /** * Get meta information about service. @@ -46,21 +41,18 @@ public interface Service extends ConfigProducer, NetworkPortRequestor { */ ServiceInfo getServiceInfo(); - /** - * @return The hostname on which this service runs. - */ + /** Returns the hostname on which this service runs. */ String getHostName(); /** Optional JVM execution options for this service */ String getJvmOptions(); /** - * Computes and returns the i'th port for this service, based on - * this Service's baseport. + * Computes and returns the i'th port for this service, based on this Service's baseport. * - * @param i The offset from 'basePort' of the port to return - * @return the i'th port relative to the base port. - * @throws IllegalStateException if i is out of range. + * @param i the offset from 'basePort' of the port to return + * @return the i'th port relative to the base port + * @throws IllegalStateException if i is out of range */ int getRelativePort(int i); @@ -70,23 +62,16 @@ public interface Service extends ConfigProducer, NetworkPortRequestor { * * @param key a key used for lookup in the service properties * @param defStr default String value returned if no value for key found - * @return the associated String value for the given key, or + * @return the associated String value for the given key */ String getServicePropertyString(String key, String defStr); int getHealthPort(); - /** - * - * @return HashMap of default dimensions for metrics. - */ + /** Returns a HashMap of default dimensions for metrics. */ HashMap<String,String> getDefaultMetricDimensions(); - /** - * Return the Affinity of this service if it has. - * - * @return The {@link com.yahoo.vespa.model.Affinity} for this service. - */ + /** Returns the Affinity of this service if it has. */ Optional<Affinity> getAffinity(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index e94fa9bf040..b3ad8db0df1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -12,8 +12,8 @@ import com.yahoo.vespa.model.container.component.AccessLogComponent; */ public class LogserverContainer extends Container { - public LogserverContainer(AbstractConfigProducer parent) { - super(parent, "" + 0, 0); + public LogserverContainer(AbstractConfigProducer parent, boolean isHostedVespa) { + super(parent, "" + 0, 0, isHostedVespa); addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, ((LogserverContainerCluster) parent).getName(), true)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 9ae9a158631..0a61b5c91c6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.admin; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; @@ -25,11 +24,6 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain protected void doPrepare(DeployState deployState) { } @Override - public void getConfig(ThreadpoolConfig.Builder builder) { - builder.maxthreads(10); - } - - @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); builder.jvm.heapsize(384) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 5b3e4e1479e..8bd4506aedc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -34,7 +34,7 @@ public class ClusterControllerContainer extends Container implements private final Set<String> bundles = new TreeSet<>(); public ClusterControllerContainer(AbstractConfigProducer parent, int index, boolean runStandaloneZooKeeper, boolean isHosted) { - super(parent, "" + index, index); + super(parent, "" + index, index, isHosted); addHandler("clustercontroller-status", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StatusHandler", "/clustercontroller-status/*"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 41d9df414ea..5cee73dff1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.admin.clustercontroller; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.vespa.model.container.ContainerCluster; /** @@ -19,11 +18,6 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC } @Override - public void getConfig(ThreadpoolConfig.Builder builder) { - builder.maxthreads(10); - } - - @Override protected void doPrepare(DeployState deployState) { } protected boolean messageBusEnabled() { return false; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java index 21fa05ceeab..bdd04dc1692 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java @@ -22,7 +22,7 @@ public class MetricsNodesConfigGenerator { private static MetricsNodesConfig.Node.Builder toNodeBuilder(MetricsProxyContainer container) { var builder = new MetricsNodesConfig.Node.Builder() - .role(container.getHost().getConfigId()) + .role(container.getHost().getHost().getConfigId()) .hostname(container.getHostName()) .metricsPort(MetricsProxyContainer.BASEPORT) .metricsPath(MetricsV1Handler.VALUES_PATH); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index fccacc3210d..0a03f216913 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -40,7 +40,7 @@ public class MetricsProxyContainer extends Container implements final boolean isHostedVespa; public MetricsProxyContainer(AbstractConfigProducer parent, String hostname, int index, boolean isHostedVespa) { - super(parent, hostname, index); + super(parent, hostname, index, isHostedVespa); this.isHostedVespa = isHostedVespa; setProp("clustertype", "admin"); setProp("index", String.valueOf(index)); @@ -132,7 +132,7 @@ public class MetricsProxyContainer extends Container implements } private String getNodeRole() { - String hostConfigId = getHost().getConfigId(); + String hostConfigId = getHost().getHost().getConfigId(); if (! isHostedVespa) return hostConfigId; return getHostResource().spec().membership() .map(ClusterMembership::stringValue) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index fbf6dcfd5eb..4b9e1c302b7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -28,7 +28,6 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; -import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.VespaModel; @@ -198,11 +197,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } } - @Override - public void getConfig(ThreadpoolConfig.Builder builder) { - builder.maxthreads(10); - } - protected boolean messageBusEnabled() { return false; } private MetricSet getAdditionalDefaultMetrics() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 56b70bec24e..b492df3a251 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -199,6 +199,13 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.http.ssl.handshake.failure.unknown.rate")); metrics.add(new Metric("jdisc.http.handler.unhandled_exceptions.rate")); + + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.max", List.of("last")); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.reserved", List.of("last")); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.busy", List.of("sum", "count", "min", "max")); + addMetric(metrics, "jdisc.http.jetty.threadpool.thread.total", List.of("sum", "count", "min", "max")); + addMetric(metrics, "jdisc.http.jetty.threadpool.queue.size", List.of("sum", "count", "min", "max")); + return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index fa72a4965b0..cc4653fc183 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.application.validation.change.ClusterSizeReductionValidator; @@ -17,6 +18,7 @@ import com.yahoo.vespa.model.application.validation.change.ContentTypeRemovalVal import com.yahoo.vespa.model.application.validation.change.GlobalDocumentChangeValidator; import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeValidator; +import com.yahoo.vespa.model.application.validation.change.NodeResourceChangeValidator; import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; @@ -27,6 +29,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; @@ -69,9 +73,11 @@ public class Validation { validateFirstTimeDeployment(model, deployState); } else { Optional<Model> currentActiveModel = deployState.getPreviousModel(); - if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) + if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) { result = validateChanges((VespaModel) currentActiveModel.get(), model, deployState.validationOverrides(), deployState.getDeployLogger(), deployState.now()); + deferConfigChangesForClustersToBeRestarted(result, model); + } } return result; } @@ -91,6 +97,7 @@ public class Validation { new ClusterSizeReductionValidator(), new ResourcesReductionValidator(), new ContainerRestartValidator(), + new NodeResourceChangeValidator() }; return Arrays.stream(validators) .flatMap(v -> v.validate(currentModel, nextModel, overrides, now).stream()) @@ -101,4 +108,21 @@ public class Validation { new AccessControlOnFirstDeploymentValidator().validate(model, deployState); } + private static void deferConfigChangesForClustersToBeRestarted(List<ConfigChangeAction> actions, VespaModel model) { + Set<ClusterSpec.Id> clustersToBeRestarted = actions.stream() + .filter(action -> action.getType() == ConfigChangeAction.Type.RESTART) + .filter(action -> action.clusterId() != null) // TODO: Remove this line after October 2020 + .map(action -> action.clusterId()) + .collect(Collectors.toSet()); + for (var clusterToRestart : clustersToBeRestarted) { + var containerCluster = model.getContainerClusters().get(clusterToRestart.value()); + if (containerCluster != null) + containerCluster.deferChangesUntilRestart(); + + var contentCluster = model.getContentClusters().get(clusterToRestart.value()); + if (contentCluster != null) + contentCluster.deferChangesUntilRestart(); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java index bec7fd1518f..b720cc13f42 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java @@ -12,7 +12,6 @@ import java.util.List; * Interface for validating changes between a current active and next config model. * * @author geirst - * @since 2014-11-18 */ public interface ChangeValidator { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java index 162f6798462..f223ba69f61 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java @@ -7,11 +7,8 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.content.cluster.ContentCluster; import java.time.Instant; -import java.util.Collections; import java.util.List; /** @@ -33,7 +30,7 @@ public class ClusterSizeReductionValidator implements ChangeValidator { overrides, now); } - return Collections.emptyList(); + return List.of(); } private void validate(Capacity current, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java index d94cd57357d..a11628bb0ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.RestartConfigs; @@ -44,32 +45,32 @@ public class ConfigValueChangeValidator implements ChangeValidator { return findConfigChangesFromModels(currentModel, nextModel).collect(Collectors.toList()); } - public Stream<ConfigChangeAction> findConfigChangesFromModels( - AbstractConfigProducerRoot currentModel, - AbstractConfigProducerRoot nextModel) { + public Stream<ConfigChangeAction> findConfigChangesFromModels(AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { return nextModel.getDescendantServices().stream() .map(service -> findConfigChangeActionForService(service, currentModel, nextModel)) .filter(Optional::isPresent) .map(Optional::get); } - private Optional<ConfigChangeAction> findConfigChangeActionForService( - Service service, - AbstractConfigProducerRoot currentModel, - AbstractConfigProducerRoot nextModel) { + private Optional<ConfigChangeAction> findConfigChangeActionForService(Service service, + AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { List<ChangesRequiringRestart> changes = findConfigChangesForService(service, currentModel, nextModel) .collect(Collectors.toList()); if (changes.isEmpty()) { return Optional.empty(); } String description = createDescriptionOfConfigChanges(changes.stream()); - return Optional.of(new VespaRestartAction(description, service.getServiceInfo())); + ClusterSpec.Id id = service.getHost().spec().membership().isPresent() ? + service.getHost().spec().membership().get().cluster().id() : + ClusterSpec.Id.from(service.getConfigId()); + return Optional.of(new VespaRestartAction(id, description, service.getServiceInfo())); } - private Stream<ChangesRequiringRestart> findConfigChangesForService( - Service service, - AbstractConfigProducerRoot currentModel, - AbstractConfigProducerRoot nextModel) { + private Stream<ChangesRequiringRestart> findConfigChangesForService(Service service, + AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { Class<? extends Service> serviceClass = service.getClass(); if (!currentModel.getService(service.getConfigId()).isPresent()) { // Service does not exist in the current model. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java index 3176ad9f912..17e4f031f3e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java @@ -2,18 +2,24 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.QrConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.vespa.model.container.ApplicationContainer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; import java.time.Instant; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; /** - * Returns a restart action for each container that has turned on {@link QrConfig#restartOnDeploy}. + * Returns a restart action for each container that has turned on {@link QrConfig#restartOnDeploy()}. * * @author bjorncs */ @@ -22,16 +28,19 @@ public class ContainerRestartValidator implements ChangeValidator { @Override public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides ignored, Instant now) { - return nextModel.getContainerClusters().values().stream() - .flatMap(cluster -> cluster.getContainers().stream()) - .filter(container -> isExistingContainer(container, currentModel)) - .filter(container -> shouldContainerRestartOnDeploy(container, nextModel)) - .map(ContainerRestartValidator::createConfigChangeAction) - .collect(toList()); + List<ConfigChangeAction> actions = new ArrayList<>(); + for (ContainerCluster<ApplicationContainer> cluster : nextModel.getContainerClusters().values()) { + actions.addAll(cluster.getContainers().stream() + .filter(container -> isExistingContainer(container, currentModel)) + .filter(container -> shouldContainerRestartOnDeploy(container, nextModel)) + .map(container -> createConfigChangeAction(cluster.id(), container)) + .collect(Collectors.toList())); + } + return actions; } - private static ConfigChangeAction createConfigChangeAction(Container container) { - return new VespaRestartAction(createMessage(container), container.getServiceInfo(), true); + private static ConfigChangeAction createConfigChangeAction(ClusterSpec.Id id, Container container) { + return new VespaRestartAction(id, createMessage(container), container.getServiceInfo(), true); } private static String createMessage(Container container) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java index 866f647a351..4a43a30c167 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java @@ -8,7 +8,6 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.cluster.ContentCluster; import java.time.Instant; -import java.util.Collections; import java.util.List; /** @@ -30,7 +29,7 @@ public class ContentClusterRemovalValidator implements ChangeValidator { now); } - return Collections.emptyList(); + return List.of(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidator.java index a691c8bb5c4..f3a018f2cdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidator.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.cluster.ContentCluster; import java.time.Instant; -import java.util.Collections; import java.util.List; /** @@ -36,7 +35,7 @@ public class ContentTypeRemovalValidator implements ChangeValidator { } } } - return Collections.emptyList(); + return List.of(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java index 198030d1f44..0b3f865c760 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java @@ -8,7 +8,6 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import java.util.Collections; import java.time.Instant; import java.util.List; import java.util.Map; @@ -30,7 +29,7 @@ public class GlobalDocumentChangeValidator implements ChangeValidator { validateContentCluster(currentEntry.getValue(), nextCluster); } } - return Collections.emptyList(); + return List.of(); } private void validateContentCluster(ContentCluster currentCluster, ContentCluster nextCluster) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java index 4ed28331467..b321d5f3fd7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java @@ -20,7 +20,6 @@ import java.util.stream.Collectors; * Validates the changes between all current and next indexed search clusters in a vespa model. * * @author geirst - * @since 2014-11-18 */ public class IndexedSearchClusterChangeValidator implements ChangeValidator { @@ -40,9 +39,7 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { ContentCluster nextCluster, ValidationOverrides overrides, Instant now) { - List<ConfigChangeAction> result = new ArrayList<>(); - result.addAll(validateDocumentDatabases(currentCluster, nextCluster, overrides, now)); - return result; + return validateDocumentDatabases(currentCluster, nextCluster, overrides, now); } private static List<ConfigChangeAction> validateDocumentDatabases(ContentCluster currentCluster, @@ -72,7 +69,7 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { NewDocumentType currentDocType = currentCluster.getDocumentDefinitions().get(docTypeName); NewDocumentType nextDocType = nextCluster.getDocumentDefinitions().get(docTypeName); List<VespaConfigChangeAction> result = - new DocumentDatabaseChangeValidator(currentDb, currentDocType, nextDb, nextDocType).validate(overrides, now); + new DocumentDatabaseChangeValidator(currentCluster.id(), currentDb, currentDocType, nextDb, nextDocType).validate(overrides, now); return modifyActions(result, getSearchNodeServices(nextCluster.getSearch().getIndexed()), docTypeName); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java index 908809b4b44..08b4b37968c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -42,7 +42,8 @@ public class IndexingModeChangeValidator implements ChangeValidator { if (currentClusterIsIndexed == nextClusterIsIndexed) return Optional.empty(); - return Optional.of(VespaRefeedAction.of(ValidationId.indexModeChange.value(), + return Optional.of(VespaRefeedAction.of(currentCluster.id(), + ValidationId.indexModeChange.value(), overrides, "Cluster '" + currentCluster.getName() + "' changed indexing mode from '" + indexingMode(currentClusterIsIndexed) + "' to '" + indexingMode(nextClusterIsIndexed) + "'", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java new file mode 100644 index 00000000000..0add9f243fe --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java @@ -0,0 +1,82 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Emits restart change actions for clusters where the node resources are changed in a way + * which requires a "restart" (container recreation) to take effect. + * Nodes will restart on their own on this condition but we want to emit restart actions to + * defer applying new config until restart. + * + * @author bratseth + */ +public class NodeResourceChangeValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) { + var restartActions = new ArrayList<ConfigChangeAction>(); + for (ClusterSpec.Id clusterId : current.allClusters()) { + Optional<NodeResources> currentResources = resourcesOf(clusterId, current); + Optional<NodeResources> nextResources = resourcesOf(clusterId, next); + if (currentResources.isEmpty() || nextResources.isEmpty()) continue; // new or removed cluster + if ( changeRequiresRestart(currentResources.get(), nextResources.get())) + restartActions.addAll(createRestartActionsFor(clusterId, current)); + } + return restartActions; + } + + private boolean changeRequiresRestart(NodeResources currentResources, NodeResources nextResources) { + return currentResources.memoryGb() != nextResources.memoryGb(); + } + + private Optional<NodeResources> resourcesOf(ClusterSpec.Id clusterId, VespaModel model) { + return model.allocatedHosts().getHosts().stream().filter(host -> host.membership().isPresent()) + .filter(host -> host.membership().get().cluster().id().equals(clusterId)) + .findFirst() + .map(host -> host.realResources()); + } + + private List<ConfigChangeAction> createRestartActionsFor(ClusterSpec.Id clusterId, VespaModel model) { + ApplicationContainerCluster containerCluster = model.getContainerClusters().get(clusterId.value()); + if (containerCluster != null) + return createRestartActionsFor(containerCluster); + + ContentCluster contentCluster = model.getContentClusters().get(clusterId.value()); + if (contentCluster != null) + return createRestartActionsFor(contentCluster); + + return List.of(); + } + + private List<ConfigChangeAction> createRestartActionsFor(ApplicationContainerCluster cluster) { + return cluster.getContainers().stream() + .map(container -> new VespaRestartAction(cluster.id(), + "Node resource change", + container.getServiceInfo(), + false)) + .collect(Collectors.toList()); + } + + private List<ConfigChangeAction> createRestartActionsFor(ContentCluster cluster) { + return cluster.getSearch().getSearchNodes().stream() + .map(node -> new VespaRestartAction(cluster.id(), + "Node resource change", + node.getServiceInfo(), + false)) + .collect(Collectors.toList()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 24b7b0949f6..0fdfcd78323 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -1,22 +1,17 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; -import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.VespaModel; import java.time.Instant; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java index 840ab69ba08..54ce869eba1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationOverrides; @@ -28,11 +29,11 @@ public class StartupCommandChangeValidator implements ChangeValidator { return findServicesWithChangedStartupCommmand(currentModel, nextModel).collect(Collectors.toList()); } - public Stream<ConfigChangeAction> findServicesWithChangedStartupCommmand( - AbstractConfigProducerRoot currentModel, AbstractConfigProducerRoot nextModel) { + public Stream<ConfigChangeAction> findServicesWithChangedStartupCommmand(AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { return nextModel.getDescendantServices().stream() .map(nextService -> currentModel.getService(nextService.getConfigId()) - .flatMap(currentService -> compareStartupCommand(currentService, nextService))) + .flatMap(currentService -> compareStartupCommand(currentService, nextService))) .filter(Optional::isPresent) .map(Optional::get); } @@ -41,13 +42,13 @@ public class StartupCommandChangeValidator implements ChangeValidator { String currentCommand = currentService.getStartupCommand(); String nextCommand = nextService.getStartupCommand(); - // Objects.equals is null-aware - if (Objects.equals(currentCommand, nextCommand)) { - return Optional.empty(); - } + if (Objects.equals(currentCommand, nextCommand)) return Optional.empty(); + String message = String.format("Startup command for '%s' has changed.\nNew command: %s.\nOld command: %s.", currentService.getServiceName(), nextCommand, currentCommand); - return Optional.of(new VespaRestartAction(message, currentService.getServiceInfo())); + return Optional.of(new VespaRestartAction(ClusterSpec.Id.from(currentService.getConfigId()), + message, + currentService.getServiceInfo())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index a96275010bf..d85d9bd2db5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.document.Attribute; @@ -59,45 +60,51 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { Instant now) { List<VespaConfigChangeAction> result = new ArrayList<>(); - result.addAll(validateDocumentTypeChanges(getDocumentType(currentCluster, currentStreamingCluster), - getDocumentType(nextCluster, nextStreamingCluster), overrides, now)); - result.addAll(validateAttributeFastAccessAdded(currentStreamingCluster.getSdConfig().getAttributeFields(), - nextStreamingCluster.getSdConfig().getAttributeFields())); - result.addAll(validateAttributeFastAccessRemoved(currentStreamingCluster.getSdConfig().getAttributeFields(), - nextStreamingCluster.getSdConfig().getAttributeFields())); + result.addAll(validateDocumentTypeChanges(currentCluster.id(), + getDocumentType(currentCluster, currentStreamingCluster), + getDocumentType(nextCluster, nextStreamingCluster), overrides, now)); + result.addAll(validateAttributeFastAccessAdded(currentCluster.id(), + currentStreamingCluster.getSdConfig().getAttributeFields(), + nextStreamingCluster.getSdConfig().getAttributeFields())); + result.addAll(validateAttributeFastAccessRemoved(currentCluster.id(), + currentStreamingCluster.getSdConfig().getAttributeFields(), + nextStreamingCluster.getSdConfig().getAttributeFields())); return modifyActions(result, getSearchNodeServices(nextCluster), nextStreamingCluster.getDocTypeName()); } - private static List<VespaConfigChangeAction> validateDocumentTypeChanges(NewDocumentType currentDocType, + private static List<VespaConfigChangeAction> validateDocumentTypeChanges(ClusterSpec.Id id, + NewDocumentType currentDocType, NewDocumentType nextDocType, ValidationOverrides overrides, Instant now) { - return new DocumentTypeChangeValidator(currentDocType, nextDocType).validate(overrides, now); + return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate(overrides, now); } private static NewDocumentType getDocumentType(ContentCluster cluster, StreamingSearchCluster streamingCluster) { return cluster.getDocumentDefinitions().get(streamingCluster.getDocTypeName()); } - private static List<VespaConfigChangeAction> validateAttributeFastAccessAdded(AttributeFields currentAttributes, + private static List<VespaConfigChangeAction> validateAttributeFastAccessAdded(ClusterSpec.Id id, + AttributeFields currentAttributes, AttributeFields nextAttributes) { - return validateAttributeFastAccessChanged(nextAttributes, currentAttributes, "add"); + return validateAttributeFastAccessChanged(id, nextAttributes, currentAttributes, "add"); } - private static List<VespaConfigChangeAction> validateAttributeFastAccessRemoved(AttributeFields currentAttributes, + private static List<VespaConfigChangeAction> validateAttributeFastAccessRemoved(ClusterSpec.Id id, + AttributeFields currentAttributes, AttributeFields nextAttributes) { - return validateAttributeFastAccessChanged(currentAttributes, nextAttributes, "remove"); + return validateAttributeFastAccessChanged(id, currentAttributes, nextAttributes, "remove"); } - private static List<VespaConfigChangeAction> validateAttributeFastAccessChanged(AttributeFields lhsAttributes, + private static List<VespaConfigChangeAction> validateAttributeFastAccessChanged(ClusterSpec.Id id, + AttributeFields lhsAttributes, AttributeFields rhsAttributes, String change) { return lhsAttributes.attributes().stream() .filter(attr -> attr.isFastAccess() && !hasFastAccessAttribute(attr.getName(), rhsAttributes)) - .map(attr -> new VespaRestartAction(new ChangeMessageBuilder(attr.getName()) - .addChange(change + " fast-access attribute").build())) + .map(attr -> new VespaRestartAction(id, new ChangeMessageBuilder(attr.getName()).addChange(change + " fast-access attribute").build())) .collect(Collectors.toList()); } @@ -117,7 +124,7 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { String docTypeName) { return result.stream() .map(action -> action.modifyAction("Document type '" + docTypeName + "': " + action.getMessage(), - services, docTypeName)) + services, docTypeName)) .collect(Collectors.toList()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java index ea57b88ff4e..96c1a6a7b09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java @@ -3,8 +3,10 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -12,14 +14,15 @@ import java.util.stream.Collectors; * between the current active vespa model and the next vespa model to prepare. * * @author geirst - * @since 2014-11-18 */ public abstract class VespaConfigChangeAction implements ConfigChangeAction { + private final ClusterSpec.Id id; private final String message; private final List<ServiceInfo> services; - protected VespaConfigChangeAction(String message, List<ServiceInfo> services) { + protected VespaConfigChangeAction(ClusterSpec.Id id, String message, List<ServiceInfo> services) { + this.id = id; this.message = message; this.services = services; } @@ -27,6 +30,9 @@ public abstract class VespaConfigChangeAction implements ConfigChangeAction { public abstract VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType); @Override + public ClusterSpec.Id clusterId() { return id; } + + @Override public String getMessage() { return message; } @@ -63,4 +69,5 @@ public abstract class VespaConfigChangeAction implements ConfigChangeAction { result = 31 * result + services.hashCode(); return result; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java index be43c6eddfb..b61cd7af151 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ClusterSpec; import java.time.Instant; import java.util.List; @@ -13,7 +14,6 @@ import java.util.List; * * @author geirst * @author bratseth - * @since 5.43 */ public class VespaRefeedAction extends VespaConfigChangeAction implements ConfigChangeRefeedAction { @@ -27,8 +27,8 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config private final String documentType; private final boolean allowed; - private VespaRefeedAction(String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { - super(message, services); + private VespaRefeedAction(ClusterSpec.Id id, String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { + super(id, message, services); this.name = name; this.documentType = documentType; this.allowed = allowed; @@ -36,19 +36,19 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config /** Creates a refeed action with some missing information */ // TODO: We should require document type or model its absence properly - public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, Instant now) { - return new VespaRefeedAction(name, message, List.of(), "", overrides.allows(name, now)); + public static VespaRefeedAction of(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { + return new VespaRefeedAction(id, name, message, List.of(), "", overrides.allows(name, now)); } /** Creates a refeed action */ - public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, + public static VespaRefeedAction of(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, List<ServiceInfo> services, String documentType, Instant now) { - return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name, now)); + return new VespaRefeedAction(id, name, message, services, documentType, overrides.allows(name, now)); } @Override public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { - return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed); + return new VespaRefeedAction(clusterId(), name, newMessage, newServices, documentType, allowed); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java index 3ea18cac1d6..ca3a408b2e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java @@ -3,8 +3,10 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeRestartAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import java.util.List; +import java.util.Optional; /** * Represents an action to restart services in order to handle a config change. @@ -15,27 +17,27 @@ public class VespaRestartAction extends VespaConfigChangeAction implements Confi private final boolean ignoreForInternalRedeploy; - public VespaRestartAction(String message) { - this(message, List.of()); + public VespaRestartAction(ClusterSpec.Id id, String message) { + this(id, message, List.of()); } - public VespaRestartAction(String message, ServiceInfo service) { - this(message, List.of(service)); + public VespaRestartAction(ClusterSpec.Id id, String message, ServiceInfo service) { + this(id, message, List.of(service)); } - public VespaRestartAction(String message, ServiceInfo services, boolean ignoreForInternalRedeploy) { - super(message, List.of(services)); + public VespaRestartAction(ClusterSpec.Id id, String message, ServiceInfo services, boolean ignoreForInternalRedeploy) { + super(id, message, List.of(services)); this.ignoreForInternalRedeploy = ignoreForInternalRedeploy; } - public VespaRestartAction(String message, List<ServiceInfo> services) { - super(message, services); + public VespaRestartAction(ClusterSpec.Id id, String message, List<ServiceInfo> services) { + super(id, message, services); this.ignoreForInternalRedeploy = false; } @Override public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { - return new VespaRestartAction(newMessage, newServices); + return new VespaRestartAction(clusterId(), newMessage, newServices); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java index 5d790c74f18..a10aac30298 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.IndexSchema; @@ -26,6 +27,7 @@ import java.util.stream.Collectors; */ public class AttributeChangeValidator { + private final ClusterSpec.Id id; private final AttributeFields currentFields; private final IndexSchema currentIndexSchema; private final NewDocumentType currentDocType; @@ -33,12 +35,14 @@ public class AttributeChangeValidator { private final IndexSchema nextIndexSchema; private final NewDocumentType nextDocType; - public AttributeChangeValidator(AttributeFields currentFields, + public AttributeChangeValidator(ClusterSpec.Id id, + AttributeFields currentFields, IndexSchema currentIndexSchema, NewDocumentType currentDocType, AttributeFields nextFields, IndexSchema nextIndexSchema, NewDocumentType nextDocType) { + this.id = id; this.currentFields = currentFields; this.currentIndexSchema = currentIndexSchema; this.currentDocType = currentDocType; @@ -60,9 +64,8 @@ public class AttributeChangeValidator { return nextFields.attributes().stream(). map(attr -> attr.getName()). filter(attrName -> !currentFields.containsAttribute(attrName) && - currentDocType.containsField(attrName)). - map(attrName -> new VespaRestartAction(new ChangeMessageBuilder(attrName). - addChange("add attribute aspect").build())). + currentDocType.containsField(attrName)). + map(attrName -> new VespaRestartAction(id, new ChangeMessageBuilder(attrName).addChange("add attribute aspect").build())). collect(Collectors.toList()); } @@ -70,10 +73,9 @@ public class AttributeChangeValidator { return currentFields.attributes().stream(). map(attr -> attr.getName()). filter(attrName -> !nextFields.containsAttribute(attrName) && - nextDocType.containsField(attrName) && - !isIndexField(attrName)). - map(attrName -> new VespaRestartAction(new ChangeMessageBuilder(attrName). - addChange("remove attribute aspect").build())). + nextDocType.containsField(attrName) && + !isIndexField(attrName)). + map(attrName -> new VespaRestartAction(id, new ChangeMessageBuilder(attrName).addChange("remove attribute aspect").build())). collect(Collectors.toList()); } @@ -90,60 +92,63 @@ public class AttributeChangeValidator { for (Attribute nextAttr : nextFields.attributes()) { Attribute currAttr = currentFields.getAttribute(nextAttr.getName()); if (currAttr != null) { - validateAttributeSetting(currAttr, nextAttr, Attribute::isFastSearch, "fast-search", result); - validateAttributeSetting(currAttr, nextAttr, Attribute::isFastAccess, "fast-access", result); - validateAttributeSetting(currAttr, nextAttr, Attribute::isHuge, "huge", result); - validateAttributeSetting(currAttr, nextAttr, Attribute::densePostingListThreshold, "dense-posting-list-threshold", result); - validateAttributeSetting(currAttr, nextAttr, Attribute::isEnabledOnlyBitVector, "rank: filter", result); - validateAttributeSetting(currAttr, nextAttr, AttributeChangeValidator::hasHnswIndex, "indexing: index", result); - validateAttributeSetting(currAttr, nextAttr, Attribute::distanceMetric, "distance-metric", result); + validateAttributeSetting(id, currAttr, nextAttr, Attribute::isFastSearch, "fast-search", result); + validateAttributeSetting(id, currAttr, nextAttr, Attribute::isFastAccess, "fast-access", result); + validateAttributeSetting(id, currAttr, nextAttr, Attribute::isHuge, "huge", result); + validateAttributeSetting(id, currAttr, nextAttr, Attribute::densePostingListThreshold, "dense-posting-list-threshold", result); + validateAttributeSetting(id, currAttr, nextAttr, Attribute::isEnabledOnlyBitVector, "rank: filter", result); + validateAttributeSetting(id, currAttr, nextAttr, AttributeChangeValidator::hasHnswIndex, "indexing: index", result); + validateAttributeSetting(id, currAttr, nextAttr, Attribute::distanceMetric, "distance-metric", result); if (hasHnswIndex(currAttr) && hasHnswIndex(nextAttr)) { - validateAttributeHnswIndexSetting(currAttr, nextAttr, HnswIndexParams::maxLinksPerNode, "max-links-per-node", result); - validateAttributeHnswIndexSetting(currAttr, nextAttr, HnswIndexParams::neighborsToExploreAtInsert, "neighbors-to-explore-at-insert", result); + validateAttributeHnswIndexSetting(id, currAttr, nextAttr, HnswIndexParams::maxLinksPerNode, "max-links-per-node", result); + validateAttributeHnswIndexSetting(id, currAttr, nextAttr, HnswIndexParams::neighborsToExploreAtInsert, "neighbors-to-explore-at-insert", result); } } } return result; } - private static void validateAttributeSetting(Attribute currentAttr, Attribute nextAttr, + private static void validateAttributeSetting(ClusterSpec.Id id, + Attribute currentAttr, Attribute nextAttr, Predicate<Attribute> predicate, String setting, List<VespaConfigChangeAction> result) { - final boolean nextValue = predicate.test(nextAttr); + boolean nextValue = predicate.test(nextAttr); if (predicate.test(currentAttr) != nextValue) { String change = nextValue ? "add" : "remove"; - result.add(new VespaRestartAction(new ChangeMessageBuilder(nextAttr.getName()). - addChange(change + " attribute '" + setting + "'").build())); + result.add(new VespaRestartAction(id, new ChangeMessageBuilder(nextAttr.getName()).addChange(change + " attribute '" + setting + "'").build())); } } - private static <T> void validateAttributeSetting(Attribute currentAttr, Attribute nextAttr, + private static <T> void validateAttributeSetting(ClusterSpec.Id id, + Attribute currentAttr, Attribute nextAttr, Function<Attribute, T> settingValueProvider, String setting, List<VespaConfigChangeAction> result) { T currentValue = settingValueProvider.apply(currentAttr); T nextValue = settingValueProvider.apply(nextAttr); if ( ! Objects.equals(currentValue, nextValue)) { String message = String.format("change property '%s' from '%s' to '%s'", setting, currentValue, nextValue); - result.add(new VespaRestartAction(new ChangeMessageBuilder(nextAttr.getName()).addChange(message).build())); + result.add(new VespaRestartAction(id, new ChangeMessageBuilder(nextAttr.getName()).addChange(message).build())); } } - private static <T> void validateAttributeHnswIndexSetting(Attribute currentAttr, Attribute nextAttr, - Function<HnswIndexParams, T> settingValueProvider, String setting, + private static <T> void validateAttributeHnswIndexSetting(ClusterSpec.Id id, + Attribute currentAttr, Attribute nextAttr, + Function<HnswIndexParams, T> settingValueProvider, + String setting, List<VespaConfigChangeAction> result) { T currentValue = settingValueProvider.apply(currentAttr.hnswIndexParams().get()); T nextValue = settingValueProvider.apply(nextAttr.hnswIndexParams().get()); if (!Objects.equals(currentValue, nextValue)) { String message = String.format("change hnsw index property '%s' from '%s' to '%s'", setting, currentValue, nextValue); - result.add(new VespaRestartAction(new ChangeMessageBuilder(nextAttr.getName()).addChange(message).build())); + result.add(new VespaRestartAction(id, new ChangeMessageBuilder(nextAttr.getName()).addChange(message).build())); } } - private List<VespaConfigChangeAction> validateTensorTypes(final ValidationOverrides overrides, Instant now) { - final List<VespaConfigChangeAction> result = new ArrayList<>(); + private List<VespaConfigChangeAction> validateTensorTypes(ValidationOverrides overrides, Instant now) { + List<VespaConfigChangeAction> result = new ArrayList<>(); - for (final Attribute nextAttr : nextFields.attributes()) { - final Attribute currentAttr = currentFields.getAttribute(nextAttr.getName()); + for (Attribute nextAttr : nextFields.attributes()) { + Attribute currentAttr = currentFields.getAttribute(nextAttr.getName()); if (currentAttr != null && currentAttr.tensorType().isPresent()) { // If the tensor attribute is not present on the new attribute, it means that the data type of the attribute @@ -154,7 +159,7 @@ public class AttributeChangeValidator { // Tensor attribute has changed type if (!nextAttr.tensorType().get().equals(currentAttr.tensorType().get())) { - result.add(createTensorTypeChangedRefeedAction(currentAttr, nextAttr, overrides, now)); + result.add(createTensorTypeChangedRefeedAction(id, currentAttr, nextAttr, overrides, now)); } } } @@ -162,15 +167,13 @@ public class AttributeChangeValidator { return result; } - private static VespaRefeedAction createTensorTypeChangedRefeedAction(Attribute currentAttr, Attribute nextAttr, ValidationOverrides overrides, Instant now) { - return VespaRefeedAction.of( - "tensor-type-change", - overrides, - new ChangeMessageBuilder(nextAttr.getName()) - .addChange( - "tensor type", - currentAttr.tensorType().get().toString(), - nextAttr.tensorType().get().toString()).build(), now); + private static VespaRefeedAction createTensorTypeChangedRefeedAction(ClusterSpec.Id id, Attribute currentAttr, Attribute nextAttr, ValidationOverrides overrides, Instant now) { + return VespaRefeedAction.of(id, + "tensor-type-change", + overrides, + new ChangeMessageBuilder(nextAttr.getName()).addChange("tensor type", + currentAttr.tensorType().get().toString(), + nextAttr.tensorType().get().toString()).build(), now); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java index deae0a89c56..68a97f33dfd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; @@ -14,19 +15,21 @@ import java.util.List; * Validates the changes between a current and next document database that is part of an indexed search cluster. * * @author geirst - * @since 2014-11-18 */ public class DocumentDatabaseChangeValidator { - private DocumentDatabase currentDatabase; - private NewDocumentType currentDocType; - private DocumentDatabase nextDatabase; - private NewDocumentType nextDocType; + private final ClusterSpec.Id id; + private final DocumentDatabase currentDatabase; + private final NewDocumentType currentDocType; + private final DocumentDatabase nextDatabase; + private final NewDocumentType nextDocType; - public DocumentDatabaseChangeValidator(DocumentDatabase currentDatabase, + public DocumentDatabaseChangeValidator(ClusterSpec.Id id, + DocumentDatabase currentDatabase, NewDocumentType currentDocType, DocumentDatabase nextDatabase, NewDocumentType nextDocType) { + this.id = id; this.currentDatabase = currentDatabase; this.currentDocType = currentDocType; this.nextDatabase = nextDatabase; @@ -43,25 +46,33 @@ public class DocumentDatabaseChangeValidator { } private List<VespaConfigChangeAction> validateAttributeChanges(ValidationOverrides overrides, Instant now) { - return new AttributeChangeValidator( - currentDatabase.getDerivedConfiguration().getAttributeFields(), - currentDatabase.getDerivedConfiguration().getIndexSchema(), currentDocType, - nextDatabase.getDerivedConfiguration().getAttributeFields(), - nextDatabase.getDerivedConfiguration().getIndexSchema(), nextDocType).validate(overrides, now); + return new AttributeChangeValidator(id, + currentDatabase.getDerivedConfiguration().getAttributeFields(), + currentDatabase.getDerivedConfiguration().getIndexSchema(), currentDocType, + nextDatabase.getDerivedConfiguration().getAttributeFields(), + nextDatabase.getDerivedConfiguration().getIndexSchema(), nextDocType) + .validate(overrides, now); } private List<VespaConfigChangeAction> validateStructFieldAttributeChanges(ValidationOverrides overrides, Instant now) { - return new StructFieldAttributeChangeValidator(currentDocType, currentDatabase.getDerivedConfiguration().getAttributeFields(), - nextDocType, nextDatabase.getDerivedConfiguration().getAttributeFields()).validate(overrides, now); + return new StructFieldAttributeChangeValidator(id, + currentDocType, + currentDatabase.getDerivedConfiguration().getAttributeFields(), + nextDocType, + nextDatabase.getDerivedConfiguration().getAttributeFields()) + .validate(overrides, now); } private List<VespaConfigChangeAction> validateIndexingScriptChanges(ValidationOverrides overrides, Instant now) { - return new IndexingScriptChangeValidator(currentDatabase.getDerivedConfiguration().getSearch(), - nextDatabase.getDerivedConfiguration().getSearch()).validate(overrides, now); + return new IndexingScriptChangeValidator(id, + currentDatabase.getDerivedConfiguration().getSearch(), + nextDatabase.getDerivedConfiguration().getSearch()) + .validate(overrides, now); } private List<VespaConfigChangeAction> validateDocumentTypeChanges(ValidationOverrides overrides, Instant now) { - return new DocumentTypeChangeValidator(currentDocType, nextDocType).validate(overrides, now); + return new DocumentTypeChangeValidator(id, currentDocType, nextDocType) + .validate(overrides, now); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java index 4fd5e82e93f..b66145a10c5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.document.Field; @@ -16,12 +17,12 @@ import java.util.stream.Collectors; * Validates the changes between a current and next document type used in a document database. * * @author toregge - * @since 2014-11-25 */ public class DocumentTypeChangeValidator { - private NewDocumentType currentDocType; - private NewDocumentType nextDocType; + private final ClusterSpec.Id id; + private final NewDocumentType currentDocType; + private final NewDocumentType nextDocType; private static abstract class FieldChange { @@ -127,8 +128,10 @@ public class DocumentTypeChangeValidator { } } - public DocumentTypeChangeValidator(NewDocumentType currentDocType, + public DocumentTypeChangeValidator(ClusterSpec.Id id, + NewDocumentType currentDocType, NewDocumentType nextDocType) { + this.id = id; this.currentDocType = currentDocType; this.nextDocType = nextDocType; } @@ -137,7 +140,8 @@ public class DocumentTypeChangeValidator { return currentDocType.getAllFields().stream(). map(field -> createFieldChange(field, nextDocType)). filter(fieldChange -> fieldChange.valid() && fieldChange.changedType()). - map(fieldChange -> VespaRefeedAction.of("field-type-change", + map(fieldChange -> VespaRefeedAction.of(id, + "field-type-change", overrides, new ChangeMessageBuilder(fieldChange.fieldName()). addChange("data type", fieldChange.currentTypeName(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index b03141fa5d9..4761d1613b8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; @@ -21,14 +22,15 @@ import java.util.Optional; * Validates the indexing script changes in all fields in the current and next search model. * * @author geirst - * @since 2014-12-08 */ public class IndexingScriptChangeValidator { + private final ClusterSpec.Id id; private final Search currentSearch; private final Search nextSearch; - public IndexingScriptChangeValidator(Search currentSearch, Search nextSearch) { + public IndexingScriptChangeValidator(ClusterSpec.Id id, Search currentSearch, Search nextSearch) { + this.id = id; this.currentSearch = currentSearch; this.nextSearch = nextSearch; } @@ -53,7 +55,7 @@ public class IndexingScriptChangeValidator { ChangeMessageBuilder messageBuilder = new ChangeMessageBuilder(nextField.getName()); new IndexingScriptChangeMessageBuilder(currentSearch, currentField, nextSearch, nextField).populate(messageBuilder); messageBuilder.addChange("indexing script", currentScript.toString(), nextScript.toString()); - return Optional.of(VespaRefeedAction.of(ValidationId.indexingChange.value(), overrides, messageBuilder.build(), now)); + return Optional.of(VespaRefeedAction.of(id, ValidationId.indexingChange.value(), overrides, messageBuilder.build(), now)); } return Optional.empty(); } @@ -68,8 +70,7 @@ public class IndexingScriptChangeValidator { } private static ScriptExpression removeOutputExpressions(ScriptExpression script) { - ScriptExpression retval = (ScriptExpression) new OutputExpressionRemover().convert(script); - return retval; + return (ScriptExpression) new OutputExpressionRemover().convert(script); } private static class OutputExpressionRemover extends ExpressionConverter { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java index e3f9610d0a4..e8b2d593de6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; @@ -33,15 +34,18 @@ import java.util.stream.Collectors; */ public class StructFieldAttributeChangeValidator { + private final ClusterSpec.Id id; private final NewDocumentType currentDocType; private final AttributeFields currentAttributes; private final NewDocumentType nextDocType; private final AttributeFields nextAttributes; - public StructFieldAttributeChangeValidator(NewDocumentType currentDocType, + public StructFieldAttributeChangeValidator(ClusterSpec.Id id, + NewDocumentType currentDocType, AttributeFields currentAttributes, NewDocumentType nextDocType, AttributeFields nextAttributes) { + this.id = id; this.currentDocType = currentDocType; this.currentAttributes = currentAttributes; this.nextDocType = nextDocType; @@ -64,10 +68,8 @@ public class StructFieldAttributeChangeValidator { private List<VespaConfigChangeAction> validateAddAttributeAspect(Context current, Context next, ValidationOverrides overrides, Instant now) { return next.structFieldAttributes.stream() .filter(nextAttr -> current.hasFieldForStructFieldAttribute(nextAttr) && - !current.hasStructFieldAttribute(nextAttr)) - .map(nextAttr -> new VespaRestartAction( - new ChangeMessageBuilder(nextAttr.getName()) - .addChange("add attribute aspect").build())) + !current.hasStructFieldAttribute(nextAttr)) + .map(nextAttr -> new VespaRestartAction(id, new ChangeMessageBuilder(nextAttr.getName()).addChange("add attribute aspect").build())) .collect(Collectors.toList()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index a47c9fdb15b..12d5e8d32ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; -import java.util.logging.Level; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.admin.Admin; @@ -22,6 +21,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -105,7 +105,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { ContainerModel logserverClusterModel = new ContainerModel(context.withParent(admin).withId(logServerCluster.getSubId())); logserverClusterModel.setCluster(logServerCluster); - LogserverContainer container = new LogserverContainer(logServerCluster); + LogserverContainer container = new LogserverContainer(logServerCluster, deployState.isHosted()); container.setHostResource(hostResource); container.initService(deployState.getDeployLogger()); logServerCluster.addContainer(container); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 9018a0231db..bc2e5cc2b8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -122,13 +122,13 @@ public class ContainerDocumentApi { } private int maxPoolSize() { - double vcpu = vcpu(cluster); + double vcpu = vcpu(cluster).orElse(0); if (vcpu == 0) return FALLBACK_MAX_POOL_SIZE; return Math.max(2, (int)Math.ceil(vcpu * feedThreadPoolSizeFactor)); } private int minPoolSize() { - double vcpu = vcpu(cluster); + double vcpu = vcpu(cluster).orElse(0); if (vcpu == 0) return FALLBACK_CORE_POOL_SIZE; return Math.max(1, (int)Math.ceil(vcpu * feedThreadPoolSizeFactor * 0.5)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 232552ea4ce..cb8abb919ac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -26,7 +26,7 @@ public final class ApplicationContainer extends Container implements QrStartConf } public ApplicationContainer(AbstractConfigProducer parent, String name, boolean retired, int index, boolean isHostedVespa) { - super(parent, name, retired, index); + super(parent, name, retired, index, isHostedVespa); this.isHostedVespa = isHostedVespa; addComponent(getFS4ResourcePool()); // TODO Remove when FS4 based search protocol is gone diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 7822b03db08..93717f0f532 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -8,9 +8,9 @@ import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; -import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.handler.metrics.MetricsProxyApiConfig; import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.handler.metrics.PrometheusV1Handler; @@ -82,8 +82,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private Integer memoryPercentage = null; - public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { - super(parent, subId, name, deployState); + public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { + super(parent, configSubId, clusterId, deployState); this.tlsClientAuthority = deployState.tlsClientAuthority(); restApiGroup = new ConfigProducerGroup<>(this, "rest-api"); servletGroup = new ConfigProducerGroup<>(this, "servlet"); @@ -274,10 +274,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat null)))); } - @Override - public void getConfig(ThreadpoolConfig.Builder builder) { - } - public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index c6de198c06a..ca4140fe5c5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -73,19 +73,19 @@ public abstract class Container extends AbstractService implements private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler"); private final ComponentGroup<Component<?, ?>> components = new ComponentGroup<>(this, "components"); - private final JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer")); + private final JettyHttpServer defaultHttpServer; - protected Container(AbstractConfigProducer parent, String name, int index) { - this(parent, name, false, index); + protected Container(AbstractConfigProducer parent, String name, int index, boolean isHostedVespa) { + this(parent, name, false, index, isHostedVespa); } - protected Container(AbstractConfigProducer parent, String name, boolean retired, int index) { + protected Container(AbstractConfigProducer parent, String name, boolean retired, int index, boolean isHostedVespa) { super(parent, name); this.name = name; this.parent = parent; this.retired = retired; this.index = index; - + this.defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), isHostedVespa); if (getHttp() == null) { addChild(defaultHttpServer); } @@ -321,7 +321,7 @@ public abstract class Container extends AbstractService implements FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer(); if (fileDistribution != null) { - builder.configid(fileDistribution.getConfigProducer(getHost()).getConfigId()); + builder.configid(fileDistribution.getConfigProducer(getHost().getHost()).getConfigId()); } return builder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 87e8f16f88c..2d7d49f03ed 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -11,15 +11,14 @@ import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; -import com.yahoo.container.handler.ThreadPoolProvider; import com.yahoo.container.di.config.PlatformBundlesConfig; -import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.container.jdisc.state.StateHandler; @@ -100,8 +99,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> DocprocConfig.Producer, ClusterInfoConfig.Producer, RoutingProviderConfig.Producer, - ConfigserverConfig.Producer, - ThreadpoolConfig.Producer + ConfigserverConfig.Producer { /** @@ -160,9 +158,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> private String jvmGCOptions = null; private String environmentVars = null; - public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { - super(parent, subId); - this.name = name; + public ContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { + super(parent, configSubId); + this.name = clusterId; this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); @@ -170,7 +168,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); - addSimpleComponent(ThreadPoolProvider.class); + addComponent(new DefaultThreadpoolProvider(this, deployState)); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent(SecurityFilterInvoker.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); @@ -186,6 +184,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> addJaxProviders(); } + public ClusterSpec.Id id() { return ClusterSpec.Id.from(getName()); } + public void setZone(Zone zone) { this.zone = zone; } @@ -617,4 +617,12 @@ public abstract class ContainerCluster<CONTAINER extends Container> protected abstract boolean messageBusEnabled(); + /** + * Mark that the config emitted by this cluster currently should be applied by clients already running with + * a previous generation of it only by restarting the consuming processes. + */ + public void deferChangesUntilRestart() { + + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index 869248edfbf..93a02833d26 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -10,9 +10,8 @@ import com.yahoo.text.XML; import com.yahoo.vespa.model.container.component.SimpleComponent; import org.w3c.dom.Element; -import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; +import java.util.OptionalDouble; /** * Component definition for a {@link java.util.concurrent.Executor} using {@link ContainerThreadPool}. @@ -49,15 +48,11 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr protected Optional<UserOptions> userOptions() { return Optional.ofNullable(userOptions); } protected boolean hasUserOptions() { return userOptions().isPresent(); } - protected static double vcpu(ContainerCluster<?> cluster) { - List<Double> vcpus = cluster.getContainers().stream() + protected static OptionalDouble vcpu(ContainerCluster<?> cluster) { + return cluster.getContainers().stream() .filter(c -> c.getHostResource() != null && c.getHostResource().realResources() != null) - .map(c -> c.getHostResource().realResources().vcpu()) - .distinct() - .collect(Collectors.toList()); - // We can only use host resource for calculation if all container nodes in the cluster are homogeneous (in terms of vcpu) - if (vcpus.size() != 1 || vcpus.get(0) == 0) return 0; - return vcpus.get(0); + .mapToDouble(c -> c.getHostResource().realResources().vcpu()) + .max(); // Use highest vcpu as scale factor } public static class UserOptions { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java new file mode 100644 index 00000000000..17040caf5c2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -0,0 +1,52 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.ThreadPoolProvider; +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +/** + * Component definition for the jdisc default threadpool provider ({@link ThreadPoolProvider}). + * + * @author bjorncs + */ +class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolConfig.Producer { + + private final ContainerCluster<?> cluster; + private final DeployState deployState; + + DefaultThreadpoolProvider(ContainerCluster<?> cluster, DeployState deployState) { + super(new ComponentModel( + BundleInstantiationSpecification.getFromStrings( + "default-threadpool", + ThreadPoolProvider.class.getName(), + null))); + this.cluster = cluster; + this.deployState = deployState; + } + + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + if (!(cluster instanceof ApplicationContainerCluster)) { + // Container clusters such as logserver, metricsproxy and clustercontroller + int defaultWorkerThreads = 10; + builder.maxthreads(defaultWorkerThreads); + builder.corePoolSize(defaultWorkerThreads); + builder.queueSize(50); + return; + } + + double threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor(); + double vcpu = ContainerThreadpool.vcpu(cluster).orElse(0); + if (threadPoolSizeFactor <= 0 || vcpu == 0) return; + + // Configuration is currently identical to the search handler's threadpool + int workerThreads = Math.max(16, (int)Math.ceil(vcpu * threadPoolSizeFactor)); // TODO(bjorncs): reduce minimum size + builder.maxthreads(workerThreads); + builder.corePoolSize(workerThreads); + builder.queueSize((int)(workerThreads * deployState.getProperties().queueSizeFactor())); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index 1350e105406..98fde2e7859 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -15,19 +15,21 @@ import java.util.List; import static com.yahoo.component.ComponentSpecification.fromString; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.16.0 + * @author Einar M R Rosenvinge + * @author bjorncs */ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { - private List<ConnectorFactory> connectorFactories = new ArrayList<>(); + private final boolean isHostedVespa; + private final List<ConnectorFactory> connectorFactories = new ArrayList<>(); - public JettyHttpServer(ComponentId id) { + public JettyHttpServer(ComponentId id, boolean isHostedVespa) { super(new ComponentModel( new BundleInstantiationSpecification(id, fromString("com.yahoo.jdisc.http.server.jetty.JettyHttpServer"), fromString("jdisc_http_service")) )); + this.isHostedVespa = isHostedVespa; final FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(id); addChild(filterBindingsProviderComponent); inject(filterBindingsProviderComponent); @@ -56,6 +58,17 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .monitoringHandlerPaths(List.of("/state/v1", "/status.html")) .searchHandlerPaths(List.of("/search")) ); + if (isHostedVespa) { + // Proxy-protocol v1/v2 is used in hosted Vespa for remote address/port + builder.accessLog(new ServerConfig.AccessLog.Builder() + .remoteAddressHeaders(List.of()) + .remotePortHeaders(List.of())); + } else { + // TODO Vespa 8: Remove legacy Yahoo headers + builder.accessLog(new ServerConfig.AccessLog.Builder() + .remoteAddressHeaders(List.of("x-forwarded-for", "y-ra", "yahooremoteip", "client-ip")) + .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); + } } static ComponentModel providerComponentModel(final ComponentId parentId, String className) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java index 3f38b2b16fa..cc9cd61df36 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java @@ -17,7 +17,7 @@ public class JettyHttpServerBuilder extends VespaDomBuilder.DomConfigProducerBui @Override protected JettyHttpServer doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element http) { - JettyHttpServer jettyHttpServer = new JettyHttpServer(new ComponentId("jdisc-jetty")); + JettyHttpServer jettyHttpServer = new JettyHttpServer(new ComponentId("jdisc-jetty"), deployState.isHosted()); for (Element serverSpec: XML.getChildren(http, "server")) { ConnectorFactory connectorFactory = new JettyConnectorBuilder().build(deployState, ancestor, serverSpec); jettyHttpServer.addConnector(connectorFactory); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 638c02caf55..dee03fb58d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -353,7 +353,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setHttp(new Http(new FilterChains(cluster))); } if(cluster.getHttp().getHttpServer().isEmpty()) { - JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer")); + JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), cluster.isHostedVespa()); cluster.getHttp().setHttpServer(defaultHttpServer); defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", Defaults.getDefaults().vespaWebServicePort())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java index 81ab2cc1503..8438214f1fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -57,14 +57,14 @@ class SearchHandler extends ProcessingHandler<SearchChains> { if (hasUserOptions()) return; double threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor(); - double vcpu = vcpu(cluster); + double vcpu = vcpu(cluster).orElse(0); if (threadPoolSizeFactor <= 0 || vcpu == 0) { builder.maxThreads(500); builder.minThreads(500); builder.queueSize(0); } else { // Controls max number of concurrent requests per container - int workerThreads = Math.max(2, (int)Math.ceil(vcpu * threadPoolSizeFactor)); + int workerThreads = Math.max(16, (int)Math.ceil(vcpu * threadPoolSizeFactor)); // TODO(bjorncs): reduce minimum size builder.maxThreads(workerThreads); builder.minThreads(workerThreads); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 9923cb2fece..1a994cd4636 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -91,7 +91,7 @@ public class ContentCluster extends AbstractConfigProducer implements private Redundancy redundancy; private ClusterControllerConfig clusterControllerConfig; private PersistenceEngine.PersistenceFactory persistenceFactory; - private final String clusterName; + private final String clusterId; private Integer maxNodesPerMerge; private final Zone zone; @@ -138,7 +138,7 @@ public class ContentCluster extends AbstractConfigProducer implements c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement); c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, redundancyBuilder, c); - validateThatGroupSiblingsAreUnique(c.clusterName, c.rootGroup); + validateThatGroupSiblingsAreUnique(c.clusterId, c.rootGroup); c.search.handleRedundancy(c.redundancy); setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); @@ -163,7 +163,7 @@ public class ContentCluster extends AbstractConfigProducer implements if (context.getParentProducer().getRoot() == null) return c; - addClusterControllers(containers, context, c.rootGroup, contentElement, c.clusterName, c); + addClusterControllers(containers, context, c.rootGroup, contentElement, c.clusterId, c); return c; } @@ -492,19 +492,21 @@ public class ContentCluster extends AbstractConfigProducer implements } - private ContentCluster(AbstractConfigProducer parent, String clusterName, + private ContentCluster(AbstractConfigProducer parent, String clusterId, Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, String routingSelection, Zone zone, boolean isHosted) { - super(parent, clusterName); + super(parent, clusterId); this.isHosted = isHosted; - this.clusterName = clusterName; + this.clusterId = clusterId; this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; this.documentSelection = routingSelection; this.zone = zone; } + public ClusterSpec.Id id() { return ClusterSpec.Id.from(clusterId); } + public void prepare(DeployState deployState) { search.prepare(); @@ -525,7 +527,7 @@ public class ContentCluster extends AbstractConfigProducer implements return clusterId != null ? clusterId : "content"; } - public String getName() { return clusterName; } + public String getName() { return clusterId; } public String getRoutingSelector() { return documentSelection; } @@ -740,4 +742,13 @@ public class ContentCluster extends AbstractConfigProducer implements builder.documenttype(docTypeBuilder); } } + + /** + * Mark that the config emitted by this cluster currently should be applied by clients already running with + * a previous generation of it only by restarting the consuming processes. + */ + public void deferChangesUntilRestart() { + + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index 981ce1bc004..bacb22b0b89 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -45,13 +45,12 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> } } - private Integer bucketMoverMaxFillAboveAverage = null; - private String clusterName; - private FileStorProducer fileStorProducer; - private IntegrityCheckerProducer integrityCheckerProducer; - private StorServerProducer storServerProducer; - private StorVisitorProducer storVisitorProducer; - private PersistenceProducer persistenceProducer; + private final String clusterName; + private final FileStorProducer fileStorProducer; + private final IntegrityCheckerProducer integrityCheckerProducer; + private final StorServerProducer storServerProducer; + private final StorVisitorProducer storVisitorProducer; + private final PersistenceProducer persistenceProducer; StorageCluster(AbstractConfigProducer parent, String clusterName, @@ -71,9 +70,6 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> @Override public void getConfig(StorBucketmoverConfig.Builder builder) { - if (bucketMoverMaxFillAboveAverage != null) { - builder.max_target_fill_rate_above_average(bucketMoverMaxFillAboveAverage); - } } @Override @@ -127,4 +123,5 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> public void getConfig(StorFilestorConfig.Builder builder) { fileStorProducer.getConfig(builder); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index b44040f843d..534bab4677c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -259,7 +259,7 @@ public class SearchNode extends AbstractService implements public void getConfig(FiledistributorrpcConfig.Builder builder) { FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer(); if (fileDistribution != null) { - FileDistributionConfigProvider configProducer = fileDistribution.getConfigProducer(getHost()); + FileDistributionConfigProvider configProducer = fileDistribution.getConfigProducer(getHost().getHost()); configProducer.getConfig(builder); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 1be58564d1b..26d8a9b0eca 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -444,12 +444,14 @@ public class ModelProvisioningTest { // Check container cluster assertEquals(1, model.getContainerClusters().size()); - Set<com.yahoo.vespa.model.Host> containerHosts = model.getContainerClusters().get("foo").getContainers().stream().map(Container::getHost).collect(Collectors.toSet()); + Set<HostResource> containerHosts = model.getContainerClusters().get("foo").getContainers().stream() + .map(Container::getHost) + .collect(Collectors.toSet()); assertEquals(10, containerHosts.size()); // Check admin clusters Admin admin = model.getAdmin(); - Set<com.yahoo.vespa.model.Host> slobrokHosts = admin.getSlobroks().stream().map(Slobrok::getHost).collect(Collectors.toSet()); + Set<HostResource> slobrokHosts = admin.getSlobroks().stream().map(Slobrok::getHost).collect(Collectors.toSet()); assertEquals(3, slobrokHosts.size()); assertTrue("Slobroks are assigned from container nodes", containerHosts.containsAll(slobrokHosts)); assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost())); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java index 65ac6c7625e..4fd2609eb67 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ClusterSpec; import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -14,25 +16,25 @@ import static org.junit.Assert.assertThat; public class ConfigChangeTestUtils { - public static VespaConfigChangeAction newRestartAction(String message) { - return new VespaRestartAction(message); + public static VespaConfigChangeAction newRestartAction(ClusterSpec.Id id, String message) { + return new VespaRestartAction(id, message); } - public static VespaConfigChangeAction newRestartAction(String message, List<ServiceInfo> services) { - return new VespaRestartAction(message, services); + public static VespaConfigChangeAction newRestartAction(ClusterSpec.Id id, String message, List<ServiceInfo> services) { + return new VespaRestartAction(id, message, services); } - public static VespaConfigChangeAction newRefeedAction(String name, String message) { - return VespaRefeedAction.of(name, ValidationOverrides.empty, message, Instant.now()); + public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, String message) { + return VespaRefeedAction.of(id, name, ValidationOverrides.empty, message, Instant.now()); } - public static VespaConfigChangeAction newRefeedAction(String name, ValidationOverrides overrides, String message, Instant now) { - return VespaRefeedAction.of(name, overrides, message, now); + public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { + return VespaRefeedAction.of(id, name, overrides, message, now); } - public static VespaConfigChangeAction newRefeedAction(String name, ValidationOverrides overrides, String message, + public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, List<ServiceInfo> services, String documentType, Instant now) { - return VespaRefeedAction.of(name, overrides, message, services, documentType, now); + return VespaRefeedAction.of(id, name, overrides, message, services, documentType, now); } public static List<ConfigChangeAction> normalizeServicesInActions(List<ConfigChangeAction> result) { @@ -53,9 +55,11 @@ public class ConfigChangeTestUtils { } public static void assertEqualActions(List<ConfigChangeAction> exp, List<ConfigChangeAction> act) { - exp.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); - act.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); - assertThat(act, equalTo(exp)); + var mutableExp = new ArrayList<>(exp); + var mutableAct = new ArrayList<>(act); + mutableExp.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); + mutableAct.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); + assertThat(mutableAct, equalTo(mutableExp)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java index 80127ac6854..8d365f24c7f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; @@ -11,15 +12,13 @@ import com.yahoo.vespa.model.content.utils.SchemaBuilder; import org.junit.Test; import java.time.Instant; -import java.util.Arrays; import java.util.List; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.assertEqualActions; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.normalizeServicesInActions; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class IndexedSearchClusterChangeValidatorTest { @@ -40,9 +39,9 @@ public class IndexedSearchClusterChangeValidatorTest { public static VespaModel newOneDocModel(String sdContent) { return new ApplicationPackageBuilder(). - addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")). - addSchemas(new SchemaBuilder(). - name("d1").content(sdContent).build()).buildCreator().create(); + addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")) + .addSchemas(new SchemaBuilder().name("d1").content(sdContent).build()) + .buildCreator().create(); } public static Fixture newTwoDocFixture(String currentSd, String nextSd) { @@ -51,11 +50,9 @@ public class IndexedSearchClusterChangeValidatorTest { public static VespaModel newTwoDocModel(String d1Content, String d2Content) { return new ApplicationPackageBuilder(). - addCluster(new ContentClusterBuilder().name("foo").docTypes("d1", "d2")). - addSchemas(new SchemaBuilder(). - name("d1").content(d1Content).build()). - addSchemas(new SchemaBuilder(). - name("d2").content(d2Content).build()). + addCluster(new ContentClusterBuilder().name("foo").docTypes("d1", "d2")) + .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build()) + .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build()). buildCreator().create(); } @@ -66,25 +63,23 @@ public class IndexedSearchClusterChangeValidatorTest { public static VespaModel newTwoClusterModel(String d1Content, String d2Content) { return new ApplicationPackageBuilder(). addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")). - addCluster(new ContentClusterBuilder().name("bar").docTypes("d2")). - addSchemas(new SchemaBuilder(). - name("d1").content(d1Content).build()). - addSchemas(new SchemaBuilder(). - name("d2").content(d2Content).build()). + addCluster(new ContentClusterBuilder().name("bar").docTypes("d2")) + .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build()) + .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build()). buildCreator().create(); } private List<ConfigChangeAction> validate() { return normalizeServicesInActions(validator.validate(currentModel, nextModel, - ValidationOverrides.empty, Instant.now())); + ValidationOverrides.empty, Instant.now())); } public void assertValidation() { - assertThat(validate().size(), is(0)); + assertTrue(validate().isEmpty()); } public void assertValidation(ConfigChangeAction exp) { - assertValidation(Arrays.asList(exp)); + assertValidation(List.of(exp)); } public void assertValidation(List<ConfigChangeAction> exp) { @@ -97,31 +92,34 @@ public class IndexedSearchClusterChangeValidatorTest { static String ATTRIBUTE_CHANGE_MSG = "Field 'f1' changed: add attribute aspect"; static String INT_FIELD = "field f1 type int { indexing: summary }"; static String FIELD_TYPE_CHANGE_MSG = "Field 'f1' changed: data type: 'string' -> 'int'"; - private static List<ServiceInfo> FOO_SERVICE = Arrays.asList( + private static final List<ServiceInfo> FOO_SERVICE = List.of( new ServiceInfo("searchnode", "null", null, null, "foo/search/cluster.foo/0", "null")); - private static List<ServiceInfo> BAR_SERVICE = Arrays.asList( + private static final List<ServiceInfo> BAR_SERVICE = List.of( new ServiceInfo("searchnode2", "null", null, null, "bar/search/cluster.bar/0", "null")); @Test public void requireThatDocumentDatabaseChangeIsDiscovered() { Fixture.newOneDocFixture(STRING_FIELD, ATTRIBUTE_FIELD). - assertValidation(newRestartAction("Document type 'd1': " + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE)); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Document type 'd1': " + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE)); } @Test public void requireThatChangeInSeveralDocumentDatabasesAreDiscovered() { Fixture.newTwoDocFixture(STRING_FIELD, ATTRIBUTE_FIELD). - assertValidation(Arrays.asList(newRestartAction("Document type 'd1': " - + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE), - newRestartAction("Document type 'd2': " + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE))); + assertValidation(List.of(newRestartAction(ClusterSpec.Id.from("test"), + "Document type 'd1': " + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE), + newRestartAction(ClusterSpec.Id.from("test"), + "Document type 'd2': " + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE))); } @Test public void requireThatChangeInSeveralContentClustersAreDiscovered() { Fixture.newTwoClusterFixture(STRING_FIELD, ATTRIBUTE_FIELD). - assertValidation(Arrays.asList(newRestartAction("Document type 'd1': " - + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE), - newRestartAction("Document type 'd2': " + ATTRIBUTE_CHANGE_MSG, BAR_SERVICE))); + assertValidation(List.of(newRestartAction(ClusterSpec.Id.from("test"), + "Document type 'd1': " + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE), + newRestartAction(ClusterSpec.Id.from("test"), + "Document type 'd2': " + ATTRIBUTE_CHANGE_MSG, BAR_SERVICE))); } @Test @@ -147,9 +145,10 @@ public class IndexedSearchClusterChangeValidatorTest { @Test public void requireThatChangingFieldTypeIsDiscovered() { Fixture f = Fixture.newOneDocFixture(STRING_FIELD, INT_FIELD); - f.assertValidation(Arrays.asList(newRefeedAction("field-type-change", - ValidationOverrides.empty, - "Document type 'd1': " + FIELD_TYPE_CHANGE_MSG, FOO_SERVICE, "d1", Instant.now()))); + f.assertValidation(List.of(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", + ValidationOverrides.empty, + "Document type 'd1': " + FIELD_TYPE_CHANGE_MSG, FOO_SERVICE, "d1", Instant.now()))); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java new file mode 100644 index 00000000000..ecf026e7d88 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java @@ -0,0 +1,129 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class NodeResourceChangeValidatorTest { + + @Test + public void test_restart_action_count() { + assertEquals(0, validate(model(1, 1, 1, 1), model(1, 1, 1, 1)).size()); + assertEquals(1, validate(model(1, 1, 1, 1), model(2, 1, 1, 1)).size()); + assertEquals(2, validate(model(1, 1, 1, 1), model(1, 2, 1, 1)).size()); + assertEquals(3, validate(model(1, 1, 1, 1), model(1, 1, 2, 1)).size()); + assertEquals(4, validate(model(1, 1, 1, 1), model(1, 1, 1, 2)).size()); + assertEquals(5, validate(model(1, 1, 1, 1), model(2, 1, 1, 2)).size()); + assertEquals(6, validate(model(1, 1, 1, 1), model(1, 2, 1, 2)).size()); + assertEquals(7, validate(model(1, 1, 1, 1), model(1, 1, 2, 2)).size()); + assertEquals(8, validate(model(1, 1, 1, 1), model(2, 1, 2, 2)).size()); + assertEquals(9, validate(model(1, 1, 1, 1), model(1, 2, 2, 2)).size()); + assertEquals(10, validate(model(1, 1, 1, 1), model(2, 2, 2, 2)).size()); + } + + @Test + public void test_restart_action_details() { + ConfigChangeAction containerAction = validate(model(1, 1, 1, 1), model(2, 1, 1, 1)).get(0); + assertEquals(ConfigChangeAction.Type.RESTART, containerAction.getType()); + assertEquals("service 'container' of type container on host0", containerAction.getServices().get(0).toString()); + assertEquals(false, containerAction.ignoreForInternalRedeploy()); + + ConfigChangeAction contentAction = validate(model(1, 1, 1, 1), model(1, 1, 2, 1)).get(0); + assertEquals(ConfigChangeAction.Type.RESTART, contentAction.getType()); + assertEquals("service 'searchnode' of type searchnode on host3", contentAction.getServices().get(0).toString()); + assertEquals(false, contentAction.ignoreForInternalRedeploy()); + } + + private List<ConfigChangeAction> validate(VespaModel current, VespaModel next) { + return new NodeResourceChangeValidator().validate(current, next, + ValidationOverrides.empty, + Clock.systemUTC().instant()); + } + + private static VespaModel model(int mem1, int mem2, int mem3, int mem4) { + var properties = new TestProperties(); + properties.setHostedVespa(true); + var deployState = new DeployState.Builder().properties(properties) + .modelHostProvisioner(new Provisioner()); + return new VespaModelCreatorWithMockPkg( + null, + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0'>\n" + + " <container id='container1' version='1.0'>\n" + + " <nodes count='1'>\n" + + " <resources vcpu='1' memory='" + mem1 + "Gb' disk='100Gb'/>" + + " </nodes>\n" + + " </container>\n" + + " <container id='container2' version='1.0'>\n" + + " <nodes count='2'>\n" + + " <resources vcpu='1' memory='" + mem2 + "Gb' disk='100Gb'/>" + + " </nodes>\n" + + " </container>\n" + + " <content id='content1' version='1.0'>\n" + + " <nodes count='3'>\n" + + " <resources vcpu='1' memory='" + mem3 + "Gb' disk='100Gb'/>" + + " </nodes>\n" + + " <documents>\n" + + " <document mode='index' type='test'/>\n" + + " </documents>\n" + + " <redundancy>2</redundancy>\n" + + " </content>\n" + + " <content id='content2' version='1.0'>\n" + + " <nodes count='4'>\n" + + " <resources vcpu='1' memory='" + mem4 + "Gb' disk='100Gb'/>" + + " </nodes>\n" + + " <documents>\n" + + " <document mode='streaming' type='test'/>\n" + + " </documents>\n" + + " <redundancy>2</redundancy>\n" + + " </content>\n" + + "</services>", + List.of("schema test { document test {} }")) + .create(deployState); + } + + private static class Provisioner implements HostProvisioner { + + private int hostsCreated = 0; + + @Override + public HostSpec allocateHost(String alias) { + return new HostSpec(alias, List.of(), Optional.empty()); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + List<HostSpec> hosts = new ArrayList<>(); + var resources = capacity.minResources().nodeResources(); + for (int i = 0; i < capacity.minResources().nodes(); i++) + hosts.add(new HostSpec("host" + (hostsCreated++), + resources, resources, resources, + ClusterMembership.from(cluster, i), + Optional.empty(), Optional.empty(), Optional.empty())); + return hosts; + } + + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java index 8edbc964bfb..f5ef50ee3a4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; @@ -162,18 +163,23 @@ public class StreamingSearchClusterChangeValidatorTest { } private static VespaConfigChangeAction createFieldTypeChangeRefeedAction(String docType, List<ServiceInfo> service) { - return ConfigChangeTestUtils.newRefeedAction("field-type-change", - ValidationOverrides.empty, + return ConfigChangeTestUtils.newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", + ValidationOverrides.empty, "Document type '" + docType + "': Field 'f1' changed: data type: 'string' -> 'int'", - service, docType, Instant.now()); + service, docType, Instant.now()); } private static VespaConfigChangeAction createAddFastAccessRestartAction() { - return ConfigChangeTestUtils.newRestartAction("Document type 'd1': Field 'f1' changed: add fast-access attribute", FOO_SERVICE); + return ConfigChangeTestUtils.newRestartAction(ClusterSpec.Id.from("test"), + "Document type 'd1': Field 'f1' changed: add fast-access attribute", + FOO_SERVICE); } private static VespaConfigChangeAction createRemoveFastAccessRestartAction() { - return ConfigChangeTestUtils.newRestartAction("Document type 'd1': Field 'f1' changed: remove fast-access attribute", FOO_SERVICE); + return ConfigChangeTestUtils.newRestartAction(ClusterSpec.Id.from("test"), + "Document type 'd1': Field 'f1' changed: remove fast-access attribute", + FOO_SERVICE); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java index e00b34d4a79..168ee797fbf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import org.junit.Test; @@ -18,12 +19,13 @@ public class AttributeChangeValidatorTest { public Fixture(String currentSd, String nextSd) throws Exception { super(currentSd, nextSd); - validator = new AttributeChangeValidator(currentDb().getDerivedConfiguration().getAttributeFields(), - currentDb().getDerivedConfiguration().getIndexSchema(), - currentDocType(), - nextDb().getDerivedConfiguration().getAttributeFields(), - nextDb().getDerivedConfiguration().getIndexSchema(), - nextDocType()); + validator = new AttributeChangeValidator(ClusterSpec.Id.from("test"), + currentDb().getDerivedConfiguration().getAttributeFields(), + currentDb().getDerivedConfiguration().getIndexSchema(), + currentDocType(), + nextDb().getDerivedConfiguration().getAttributeFields(), + nextDb().getDerivedConfiguration().getIndexSchema(), + nextDocType()); } @Override @@ -37,16 +39,16 @@ public class AttributeChangeValidatorTest { public void adding_attribute_aspect_require_restart() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary }", "field f1 type string { indexing: attribute | summary }"); - f.assertValidation(newRestartAction( - "Field 'f1' changed: add attribute aspect")); + f.assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: add attribute aspect")); } @Test public void removing_attribute_aspect_require_restart() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: attribute | summary }", "field f1 type string { indexing: summary }"); - f.assertValidation(newRestartAction( - "Field 'f1' changed: remove attribute aspect")); + f.assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: remove attribute aspect")); } @Test @@ -65,24 +67,24 @@ public class AttributeChangeValidatorTest { public void changing_fast_search_require_restart() throws Exception { new Fixture("field f1 type string { indexing: attribute }", "field f1 type string { indexing: attribute \n attribute: fast-search }"). - assertValidation(newRestartAction( - "Field 'f1' changed: add attribute 'fast-search'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: add attribute 'fast-search'")); } @Test public void changing_fast_access_require_restart() throws Exception { new Fixture("field f1 type string { indexing: attribute \n attribute: fast-access }", "field f1 type string { indexing: attribute }"). - assertValidation(newRestartAction( - "Field 'f1' changed: remove attribute 'fast-access'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: remove attribute 'fast-access'")); } @Test public void changing_huge_require_restart() throws Exception { new Fixture("field f1 type string { indexing: attribute }", "field f1 type string { indexing: attribute \n attribute: huge }"). - assertValidation(newRestartAction( - "Field 'f1' changed: add attribute 'huge'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: add attribute 'huge'")); } @Test @@ -90,8 +92,8 @@ public class AttributeChangeValidatorTest { new Fixture( "field f1 type predicate { indexing: attribute \n index { arity: 8 \n dense-posting-list-threshold: 0.2 } }", "field f1 type predicate { indexing: attribute \n index { arity: 8 \n dense-posting-list-threshold: 0.4 } }"). - assertValidation(newRestartAction( - "Field 'f1' changed: change property 'dense-posting-list-threshold' from '0.2' to '0.4'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: change property 'dense-posting-list-threshold' from '0.2' to '0.4'")); } @Test @@ -113,18 +115,18 @@ public class AttributeChangeValidatorTest { new Fixture( "field f1 type tensor(x[2]) { indexing: attribute }", "field f1 type tensor(x[3]) { indexing: attribute }") - .assertValidation(newRefeedAction( - "tensor-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: tensor type: 'tensor(x[2])' -> 'tensor(x[3])'", Instant.now())); + .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "tensor-type-change", + ValidationOverrides.empty, + "Field 'f1' changed: tensor type: 'tensor(x[2])' -> 'tensor(x[3])'", Instant.now())); new Fixture( "field f1 type tensor(x[5]) { indexing: attribute }", "field f1 type tensor(x[3]) { indexing: attribute }") - .assertValidation(newRefeedAction( - "tensor-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: tensor type: 'tensor(x[5])' -> 'tensor(x[3])'", Instant.now())); + .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "tensor-type-change", + ValidationOverrides.empty, + "Field 'f1' changed: tensor type: 'tensor(x[5])' -> 'tensor(x[3])'", Instant.now())); } @Test @@ -139,32 +141,32 @@ public class AttributeChangeValidatorTest { public void adding_rank_filter_requires_restart() throws Exception { new Fixture("field f1 type string { indexing: attribute }", "field f1 type string { indexing: attribute \n rank: filter }"). - assertValidation(newRestartAction( - "Field 'f1' changed: add attribute 'rank: filter'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: add attribute 'rank: filter'")); } @Test public void removing_rank_filter_requires_restart() throws Exception { new Fixture("field f1 type string { indexing: attribute \n rank: filter }", "field f1 type string { indexing: attribute }"). - assertValidation(newRestartAction( - "Field 'f1' changed: remove attribute 'rank: filter'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: remove attribute 'rank: filter'")); } @Test public void adding_hnsw_index_requires_restart() throws Exception { new Fixture("field f1 type tensor(x[2]) { indexing: attribute }", "field f1 type tensor(x[2]) { indexing: attribute | index \n index { hnsw } }"). - assertValidation(newRestartAction( - "Field 'f1' changed: add attribute 'indexing: index'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: add attribute 'indexing: index'")); } @Test public void removing_hnsw_index_requres_restart() throws Exception { new Fixture("field f1 type tensor(x[2]) { indexing: attribute | index \n index { hnsw } }", "field f1 type tensor(x[2]) { indexing: attribute }"). - assertValidation(newRestartAction( - "Field 'f1' changed: remove attribute 'indexing: index'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: remove attribute 'indexing: index'")); } @Test @@ -172,8 +174,9 @@ public class AttributeChangeValidatorTest { new Fixture("field f1 type tensor(x[2]) { indexing: attribute }", "field f1 type tensor(x[2]) { indexing: attribute \n attribute { " + "distance-metric: geodegrees \n } }"). - assertValidation(newRestartAction("Field 'f1' changed: change property " + - "'distance-metric' from 'EUCLIDEAN' to 'GEODEGREES'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: change property " + + "'distance-metric' from 'EUCLIDEAN' to 'GEODEGREES'")); } @Test @@ -181,8 +184,9 @@ public class AttributeChangeValidatorTest { new Fixture("field f1 type tensor(x[2]) { indexing: attribute | index \n index { hnsw } }", "field f1 type tensor(x[2]) { indexing: attribute | index \n attribute { " + "distance-metric: geodegrees \n } }"). - assertValidation(newRestartAction("Field 'f1' changed: change property " + - "'distance-metric' from 'EUCLIDEAN' to 'GEODEGREES'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: change property " + + "'distance-metric' from 'EUCLIDEAN' to 'GEODEGREES'")); } @Test @@ -190,8 +194,9 @@ public class AttributeChangeValidatorTest { new Fixture("field f1 type tensor(x[2]) { indexing: attribute | index \n index { hnsw } }", "field f1 type tensor(x[2]) { indexing: attribute | index \n index { " + "hnsw { max-links-per-node: 4 } } }"). - assertValidation(newRestartAction("Field 'f1' changed: change hnsw index property " + - "'max-links-per-node' from '16' to '4'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: change hnsw index property " + + "'max-links-per-node' from '16' to '4'")); } @Test @@ -199,7 +204,8 @@ public class AttributeChangeValidatorTest { new Fixture("field f1 type tensor(x[2]) { indexing: attribute | index \n index { hnsw } }", "field f1 type tensor(x[2]) { indexing: attribute | index \n index { " + "hnsw { neighbors-to-explore-at-insert: 100 } } }"). - assertValidation(newRestartAction("Field 'f1' changed: change hnsw index property " + - "'neighbors-to-explore-at-insert' from '200' to '100'")); + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: change hnsw index property " + + "'neighbors-to-explore-at-insert' from '200' to '100'")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java index c24b5250a5d..60b17142340 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import org.junit.Test; @@ -19,7 +20,11 @@ public class DocumentDatabaseChangeValidatorTest { public Fixture(String currentSd, String nextSd) throws Exception { super(currentSd, nextSd); - validator = new DocumentDatabaseChangeValidator(currentDb(), currentDocType(), nextDb(), nextDocType()); + validator = new DocumentDatabaseChangeValidator(ClusterSpec.Id.from("test"), + currentDb(), + currentDocType(), + nextDb(), + nextDocType()); } @Override @@ -42,13 +47,17 @@ public class DocumentDatabaseChangeValidatorTest { "field f3 type string { indexing: summary } " + "field f4 type array<s> { struct-field s1 { indexing: attribute } }"); f.assertValidation(Arrays.asList( - newRestartAction("Field 'f1' changed: add attribute aspect"), - newRestartAction("Field 'f4.s1' changed: add attribute aspect"), - newRefeedAction("indexing-change", + newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: add attribute aspect"), + newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f4.s1' changed: add attribute aspect"), + newRefeedAction(ClusterSpec.Id.from("test"), + "indexing-change", ValidationOverrides.empty, "Field 'f2' changed: add index aspect, indexing script: '{ input f2 | summary f2; }' -> " + "'{ input f2 | tokenize normalize stem:\"BEST\" | index f2 | summary f2; }'", Instant.now()), - newRefeedAction("field-type-change", + newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f3' changed: data type: 'int' -> 'string'", Instant.now()))); } @@ -56,7 +65,7 @@ public class DocumentDatabaseChangeValidatorTest { @Test public void requireThatRemovingAttributeAspectFromIndexFieldIsOk() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: index | attribute }", - "field f1 type string { indexing: index }"); + "field f1 type string { indexing: index }"); f.assertValidation(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 4bba66f0fb3..190c2c8c645 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.ReferenceDataType; @@ -34,7 +35,9 @@ public class DocumentTypeChangeValidatorTest { public Fixture(String currentSd, String nextSd) throws Exception { super(currentSd, nextSd); - validator = new DocumentTypeChangeValidator(currentDocType(), nextDocType()); + validator = new DocumentTypeChangeValidator(ClusterSpec.Id.from("test"), + currentDocType(), + nextDocType()); } @Override @@ -62,7 +65,8 @@ public class DocumentTypeChangeValidatorTest { public void requireThatDataTypeChangeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary }", "field f1 type int { indexing: summary }"); - f.assertValidation(newRefeedAction("field-type-change", + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f1' changed: data type: 'string' -> 'int'", Instant.now())); @@ -72,7 +76,8 @@ public class DocumentTypeChangeValidatorTest { public void requireThatAddingCollectionTypeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary }", "field f1 type array<string> { indexing: summary }"); - f.assertValidation(newRefeedAction("field-type-change", + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f1' changed: data type: 'string' -> 'Array<string>'", Instant.now())); } @@ -89,7 +94,8 @@ public class DocumentTypeChangeValidatorTest { public void requireThatNestedDataTypeChangeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", "field f1 type array<int> { indexing: summary }"); - f.assertValidation(newRefeedAction("field-type-change", + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f1' changed: data type: 'Array<string>' -> 'Array<int>'", Instant.now())); } @@ -98,7 +104,8 @@ public class DocumentTypeChangeValidatorTest { public void requireThatChangedCollectionTypeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", "field f1 type weightedset<string> { indexing: summary }"); - f.assertValidation(newRefeedAction("field-type-change", + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f1' changed: data type: 'Array<string>' -> 'WeightedSet<string>'", Instant.now())); } @@ -107,10 +114,12 @@ public class DocumentTypeChangeValidatorTest { public void requireThatMultipleDataTypeChangesIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary } field f2 type int { indexing: summary }" , "field f2 type string { indexing: summary } field f1 type int { indexing: summary }"); - f.assertValidation(Arrays.asList(newRefeedAction("field-type-change", + f.assertValidation(Arrays.asList(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f1' changed: data type: 'string' -> 'int'", Instant.now()), - newRefeedAction("field-type-change", + newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f2' changed: data type: 'int' -> 'string'", Instant.now()))); } @@ -147,7 +156,8 @@ public class DocumentTypeChangeValidatorTest { public void requireThatDataTypeChangeInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} } field f2 type s1 { indexing: summary }", "struct s1 { field f1 type int {} } field f2 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction("field-type-change", + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f2' changed: data type: 's1:{f1:string}' -> 's1:{f1:int}'", Instant.now())); } @@ -156,7 +166,8 @@ public class DocumentTypeChangeValidatorTest { public void requireThatNestedDataTypeChangeInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type array<string> {} } field f2 type s1 { indexing: summary }", "struct s1 { field f1 type array<int> {} } field f2 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction("field-type-change", + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f2' changed: data type: 's1:{f1:Array<string>}' -> 's1:{f1:Array<int>}'", Instant.now())); } @@ -165,7 +176,8 @@ public class DocumentTypeChangeValidatorTest { public void requireThatDataTypeChangeInNestedStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} } struct s2 { field f2 type s1 {} } field f3 type s2 { indexing: summary }", "struct s1 { field f1 type int {} } struct s2 { field f2 type s1 {} } field f3 type s2 { indexing: summary }"); - f.assertValidation(newRefeedAction("field-type-change", + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f3' changed: data type: 's2:{s1:{f1:string}}' -> 's2:{s1:{f1:int}}'", Instant.now())); } @@ -174,16 +186,17 @@ public class DocumentTypeChangeValidatorTest { public void requireThatMultipleDataTypeChangesInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} field f2 type int {} } field f3 type s1 { indexing: summary }", "struct s1 { field f1 type int {} field f2 type string {} } field f3 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction("field-type-change", + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), + "field-type-change", ValidationOverrides.empty, "Field 'f3' changed: data type: 's1:{f1:string,f2:int}' -> 's1:{f1:int,f2:string}'", Instant.now())); } @Test public void requireThatChangingTargetTypeOfReferenceFieldIsNotOK() { - DocumentTypeChangeValidator validator = new DocumentTypeChangeValidator( - createDocumentTypeWithReferenceField("oldDoc"), - createDocumentTypeWithReferenceField("newDoc")); + var validator = new DocumentTypeChangeValidator(ClusterSpec.Id.from("test"), + createDocumentTypeWithReferenceField("oldDoc"), + createDocumentTypeWithReferenceField("newDoc")); List<VespaConfigChangeAction> result = validator.validate(ValidationOverrides.empty, Instant.now()); assertEquals(1, result.size()); VespaConfigChangeAction action = result.get(0); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 7a5b235737a..8ffd02a4381 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; @@ -20,8 +21,9 @@ public class IndexingScriptChangeValidatorTest { public Fixture(String currentSd, String nextSd) throws Exception { super(currentSd, nextSd); - validator = new IndexingScriptChangeValidator(currentDb().getDerivedConfiguration().getSearch(), - nextDb().getDerivedConfiguration().getSearch()); + validator = new IndexingScriptChangeValidator(ClusterSpec.Id.from("test"), + currentDb().getDerivedConfiguration().getSearch(), + nextDb().getDerivedConfiguration().getSearch()); } @Override @@ -31,6 +33,7 @@ public class IndexingScriptChangeValidatorTest { } private static class ScriptFixture { + private final ScriptExpression currentScript; private final ScriptExpression nextScript; @@ -44,15 +47,16 @@ public class IndexingScriptChangeValidatorTest { } } - private static String FIELD = "field f1 type string"; - private static String FIELD_F2 = "field f2 type string"; + private static final String FIELD = "field f1 type string"; + private static final String FIELD_F2 = "field f2 type string"; private static VespaConfigChangeAction expectedAction(String changedMsg, String fromScript, String toScript) { return expectedAction("f1", changedMsg, fromScript, toScript); } private static VespaConfigChangeAction expectedAction(String field, String changedMsg, String fromScript, String toScript) { - return VespaRefeedAction.of("indexing-change", + return VespaRefeedAction.of(ClusterSpec.Id.from("test"), + "indexing-change", ValidationOverrides.empty, "Field '" + field + "' changed: " + (changedMsg.isEmpty() ? "" : changedMsg + ", ") + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java index 2d68284c9a5..04efffab438 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import org.junit.Test; @@ -17,16 +18,18 @@ import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTe public class StructFieldAttributeChangeValidatorTestCase { private static class Fixture extends ContentClusterFixture { - private StructFieldAttributeChangeValidator structFieldAttributeValidator; - private DocumentTypeChangeValidator docTypeValidator; + + private final StructFieldAttributeChangeValidator structFieldAttributeValidator; + private final DocumentTypeChangeValidator docTypeValidator; public Fixture(String currentSd, String nextSd) throws Exception { super(currentSd, nextSd); - structFieldAttributeValidator = new StructFieldAttributeChangeValidator(currentDocType(), - currentDb().getDerivedConfiguration().getAttributeFields(), - nextDocType(), - nextDb().getDerivedConfiguration().getAttributeFields()); - docTypeValidator = new DocumentTypeChangeValidator(currentDocType(), nextDocType()); + structFieldAttributeValidator = new StructFieldAttributeChangeValidator(ClusterSpec.Id.from("test"), + currentDocType(), + currentDb().getDerivedConfiguration().getAttributeFields(), + nextDocType(), + nextDb().getDerivedConfiguration().getAttributeFields()); + docTypeValidator = new DocumentTypeChangeValidator(ClusterSpec.Id.from("test"), currentDocType(), nextDocType()); } @Override @@ -49,34 +52,34 @@ public class StructFieldAttributeChangeValidatorTestCase { @Test public void adding_attribute_aspect_to_struct_field_requires_restart() throws Exception { validate(arrayOfStruct(oneFieldStruct(), ""), - arrayOfStruct(oneFieldStruct(), structAttribute("s1")), - newRestartAction("Field 'f1.s1' changed: add attribute aspect")); + arrayOfStruct(oneFieldStruct(), structAttribute("s1")), + newRestartAction(ClusterSpec.Id.from("test"), "Field 'f1.s1' changed: add attribute aspect")); validate(mapOfStruct(oneFieldStruct(), ""), - mapOfStruct(oneFieldStruct(), structAttribute("key")), - newRestartAction("Field 'f1.key' changed: add attribute aspect")); + mapOfStruct(oneFieldStruct(), structAttribute("key")), + newRestartAction(ClusterSpec.Id.from("test"), "Field 'f1.key' changed: add attribute aspect")); validate(mapOfStruct(oneFieldStruct(), ""), - mapOfStruct(oneFieldStruct(), structAttribute("value.s1")), - newRestartAction("Field 'f1.value.s1' changed: add attribute aspect")); + mapOfStruct(oneFieldStruct(), structAttribute("value.s1")), + newRestartAction(ClusterSpec.Id.from("test"), "Field 'f1.value.s1' changed: add attribute aspect")); validate(mapOfPrimitive(""), mapOfPrimitive(structAttribute("key")), - newRestartAction("Field 'f1.key' changed: add attribute aspect")); + newRestartAction(ClusterSpec.Id.from("test"), "Field 'f1.key' changed: add attribute aspect")); validate(mapOfPrimitive(""), mapOfPrimitive(structAttribute("value")), - newRestartAction("Field 'f1.value' changed: add attribute aspect")); + newRestartAction(ClusterSpec.Id.from("test"), "Field 'f1.value' changed: add attribute aspect")); } @Test public void removing_attribute_aspect_from_struct_field_is_ok() throws Exception { validate(arrayOfStruct(oneFieldStruct(), structAttribute("s1")), - arrayOfStruct(oneFieldStruct(), "")); + arrayOfStruct(oneFieldStruct(), "")); validate(mapOfStruct(oneFieldStruct(), structAttribute("key")), - mapOfStruct(oneFieldStruct(), "")); + mapOfStruct(oneFieldStruct(), "")); validate(mapOfStruct(oneFieldStruct(), structAttribute("value.s1")), - mapOfStruct(oneFieldStruct(), "")); + mapOfStruct(oneFieldStruct(), "")); validate(mapOfPrimitive(structAttribute("key")), mapOfPrimitive("")); @@ -89,34 +92,34 @@ public class StructFieldAttributeChangeValidatorTestCase { arrayOfStruct(twoFieldStruct(), structAttribute("s2"))); validate(mapOfStruct(oneFieldStruct(), ""), - mapOfStruct(twoFieldStruct(), structAttribute("value.s2"))); + mapOfStruct(twoFieldStruct(), structAttribute("value.s2"))); } @Test public void removing_struct_field_with_attribute_aspect_is_ok() throws Exception { validate(arrayOfStruct(twoFieldStruct(), structAttribute("s2")), - arrayOfStruct(oneFieldStruct(), "")); + arrayOfStruct(oneFieldStruct(), "")); validate(mapOfStruct(twoFieldStruct(), structAttribute("value.s2")), - mapOfStruct(oneFieldStruct(), "")); + mapOfStruct(oneFieldStruct(), "")); } @Test public void adding_struct_field_without_attribute_aspect_is_ok() throws Exception { validate(arrayOfStruct(oneFieldStruct(), ""), - arrayOfStruct(twoFieldStruct(), "")); + arrayOfStruct(twoFieldStruct(), "")); validate(mapOfStruct(oneFieldStruct(), ""), - mapOfStruct(twoFieldStruct(), "")); + mapOfStruct(twoFieldStruct(), "")); } @Test public void removing_struct_field_without_attribute_aspect_is_ok() throws Exception { validate(arrayOfStruct(twoFieldStruct(), ""), - arrayOfStruct(oneFieldStruct(), "")); + arrayOfStruct(oneFieldStruct(), "")); validate(mapOfStruct(twoFieldStruct(), ""), - mapOfStruct(oneFieldStruct(), "")); + mapOfStruct(oneFieldStruct(), "")); } private static String oneFieldStruct() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index d493afd9c1f..3515efb7bc1 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -4,9 +4,11 @@ package com.yahoo.vespa.model.container; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.RoutingProviderConfig; +import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; @@ -14,6 +16,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.container.di.config.PlatformBundlesConfig; +import com.yahoo.container.handler.ThreadPoolProvider; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; @@ -33,6 +36,7 @@ import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -167,12 +171,11 @@ public class ContainerClusterTest { assertEquals(512, qrStartConfig.jvm().heapsize()); assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); + root.freezeModelTopology(); - ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); - cluster.getConfig(tpBuilder); - ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = root.getConfig(ThreadpoolConfig.class, "container0/component/default-threadpool"); assertEquals(10, threadpoolConfig.maxthreads()); - assertEquals(0, threadpoolConfig.queueSize()); + assertEquals(50, threadpoolConfig.queueSize()); } @Test @@ -219,10 +222,9 @@ public class ContainerClusterTest { MockRoot root = new MockRoot("foo", state); ApplicationContainerCluster cluster = createContainerCluster(root, false); addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + root.freezeModelTopology(); - ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); - cluster.getConfig(tpBuilder); - ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = root.getConfig(ThreadpoolConfig.class, "container0/component/default-threadpool"); assertEquals(500, threadpoolConfig.maxthreads()); assertEquals(0, threadpoolConfig.queueSize()); } @@ -232,15 +234,52 @@ public class ContainerClusterTest { MockRoot root = new MockRoot("foo"); ApplicationContainerCluster cluster = createContainerCluster(root, false); addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + root.freezeModelTopology(); - ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); - cluster.getConfig(tpBuilder); - ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = root.getConfig(ThreadpoolConfig.class, "container0/component/default-threadpool"); assertEquals(500, threadpoolConfig.maxthreads()); assertEquals(0, threadpoolConfig.queueSize()); } @Test + public void container_cluster_has_default_threadpool_provider() { + MockRoot root = new MockRoot("foo"); + ApplicationContainerCluster cluster = createContainerCluster(root, false); + addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + root.freezeModelTopology(); + + ComponentId expectedComponentId = new ComponentId("default-threadpool"); + var components = cluster.getComponentsMap(); + assertThat(components, hasKey(expectedComponentId)); + Component<?, ?> component = components.get(expectedComponentId); + assertEquals(ThreadPoolProvider.class.getName(), component.getClassId().getName()); + } + + @Test + public void config_for_default_threadpool_provider_scales_with_node_resources() { + HostProvisionerWithCustomRealResource hostProvisioner = new HostProvisionerWithCustomRealResource(); + MockRoot root = new MockRoot( + "foo", + new DeployState.Builder() + .properties(new TestProperties() + .setThreadPoolSizeFactor(4) + .setQueueSizeFactor(20)) + .applicationPackage(new MockApplicationPackage.Builder().build()) + .modelHostProvisioner(hostProvisioner) + .build()); + ApplicationContainerCluster cluster = createContainerCluster(root, false); + HostResource hostResource = new HostResource( + new Host(null, "host-c1"), + hostProvisioner.allocateHost("host-c1")); + addContainerWithHostResource(root.deployLogger(), cluster, "c1", hostResource); + root.freezeModelTopology(); + + ThreadpoolConfig threadpoolConfig = root.getConfig(ThreadpoolConfig.class, "container0/component/default-threadpool"); + assertEquals(16, threadpoolConfig.maxthreads()); + assertEquals(320, threadpoolConfig.queueSize()); + } + + @Test public void requireThatRoutingProviderIsDisabledForNonHosted() { DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(false)).build(); MockRoot root = new MockRoot("foo", state); @@ -287,8 +326,12 @@ public class ContainerClusterTest { private static void addContainer(DeployLogger deployLogger, ApplicationContainerCluster cluster, String name, String hostName) { + addContainerWithHostResource(deployLogger, cluster, name, new HostResource(new Host(null, hostName))); + } + + private static void addContainerWithHostResource(DeployLogger deployLogger, ApplicationContainerCluster cluster, String name, HostResource hostResource) { ApplicationContainer container = new ApplicationContainer(cluster, name, 0, cluster.isHostedVespa()); - container.setHostResource(new HostResource(new Host(null, hostName))); + container.setHostResource(hostResource); container.initService(deployLogger); cluster.addContainer(container); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/HostProvisionerWithCustomRealResource.java b/config-model/src/test/java/com/yahoo/vespa/model/container/HostProvisionerWithCustomRealResource.java new file mode 100644 index 00000000000..450049ffe99 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/HostProvisionerWithCustomRealResource.java @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.provision.Host; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.net.HostName; + +import java.util.List; +import java.util.Optional; + +/** + * @author bjorncs + */ +public class HostProvisionerWithCustomRealResource implements HostProvisioner { + + @Override + public HostSpec allocateHost(String alias) { + Host host = new Host(HostName.getLocalhost()); + ClusterMembership membership = ClusterMembership.from( + ClusterSpec + .specification( + ClusterSpec.Type.container, + ClusterSpec.Id.from("id")) + .vespaVersion("") + .group(ClusterSpec.Group.from(0)) + .build(), + 0); + return new HostSpec( + host.hostname(), new NodeResources(4, 0, 0, 0), NodeResources.unspecified(), NodeResources.unspecified(), + membership, Optional.empty(), Optional.empty(), Optional.empty()); + } + + @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return List.of(); } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index def5da3a9c2..9f061dcbd0a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -1,20 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; -import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; -import com.yahoo.net.HostName; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.HostProvisionerWithCustomRealResource; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; @@ -23,9 +15,7 @@ import org.w3c.dom.Element; import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import static org.hamcrest.CoreMatchers.equalTo; @@ -158,26 +148,4 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa assertEquals(1000, feedThreadpoolConfig.queueSize()); } - private static class HostProvisionerWithCustomRealResource implements HostProvisioner { - - @Override - public HostSpec allocateHost(String alias) { - Host host = new Host(HostName.getLocalhost()); - ClusterMembership membership = ClusterMembership.from( - ClusterSpec - .specification( - ClusterSpec.Type.container, - ClusterSpec.Id.from("id")) - .vespaVersion("") - .group(ClusterSpec.Group.from(0)) - .build(), - 0); - return new HostSpec( - host.hostname(), new NodeResources(4, 0, 0, 0), NodeResources.unspecified(), NodeResources.unspecified(), - membership, Optional.empty(), Optional.empty(), Optional.empty()); - } - - @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return List.of(); } - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionLock.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionLock.java new file mode 100644 index 00000000000..633c8520e72 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionLock.java @@ -0,0 +1,32 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import com.yahoo.transaction.Mutex; + +import java.util.Objects; + +/** + * A type-safe wrapper for an application's provision lock. + * + * @author mpolden + */ +public class ProvisionLock implements AutoCloseable { + + private final ApplicationId application; + private final Mutex lock; + + public ProvisionLock(ApplicationId application, Mutex lock) { + this.application = Objects.requireNonNull(application); + this.lock = Objects.requireNonNull(lock); + } + + public ApplicationId application() { + return application; + } + + @Override + public void close() { + lock.close(); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java index 2a1528f5368..1eb2c1e61b2 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java @@ -31,17 +31,36 @@ public interface Provisioner { * @param application The {@link ApplicationId} that was activated. * @param hosts a set of {@link HostSpec}. */ + // TODO(mpolden): Remove void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts); /** + * Activates the allocation of nodes to this application captured in the hosts argument. + * + * @param transaction Transaction with operations to commit together with any operations done within the provisioner. + * @param hosts a set of {@link HostSpec}. + * @param lock A provision lock for the relevant application. This must be held when calling this. + */ + void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock); + + /** * Transactionally remove this application. * * @param transaction Transaction with operations to commit together with any operations done within the provisioner. * @param application the application to remove */ + // TODO(mpolden): Remove void remove(NestedTransaction transaction, ApplicationId application); /** + * Transactionally remove application guarded by given lock. + * + * @param transaction Transaction with operations to commit together with any operations done within the provisioner. + * @param lock A provision lock for the relevant application. This must be held when calling this. + */ + void remove(NestedTransaction transaction, ProvisionLock lock); + + /** * Requests a restart of the services of the given application * * @param application the application to restart @@ -49,4 +68,7 @@ public interface Provisioner { */ void restart(ApplicationId application, HostFilter filter); + /** Returns a provision lock for the given application */ + ProvisionLock lock(ApplicationId application); + } diff --git a/config/src/tests/api/api.cpp b/config/src/tests/api/api.cpp index 0af2b848ea5..a48f36aea9a 100644 --- a/config/src/tests/api/api.cpp +++ b/config/src/tests/api/api.cpp @@ -1,13 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> +#include <vespa/config/common/configcontext.h> #include <config-my.h> using namespace config; TEST("require that can subscribe with empty config id") { ConfigSet set; - ConfigContext::SP ctx(new ConfigContext(set)); + auto ctx = std::make_shared<ConfigContext>(set); MyConfigBuilder builder; builder.myField = "myfoo"; set.addBuilder("", &builder); diff --git a/config/src/tests/configfetcher/configfetcher.cpp b/config/src/tests/configfetcher/configfetcher.cpp index 856b1198ce8..508322bb74e 100644 --- a/config/src/tests/configfetcher/configfetcher.cpp +++ b/config/src/tests/configfetcher/configfetcher.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/helper/configfetcher.h> +#include <vespa/config/common/configcontext.h> #include <vespa/vespalib/util/exception.h> #include "config-my.h" #include <atomic> @@ -12,7 +13,7 @@ class MyCallback : public IFetcherCallback<MyConfig> { public: MyCallback(const std::string & badConfig=""); - ~MyCallback(); + ~MyCallback() override; void configure(std::unique_ptr<MyConfig> config) override { _config = std::move(config); @@ -126,7 +127,7 @@ struct ConfigFixture { ConfigContext::SP context; ConfigFixture() : builder(), set(), context() { set.addBuilder("cfgid", &builder); - context.reset(new ConfigContext(set)); + context = std::make_shared<ConfigContext>(set); } }; diff --git a/config/src/tests/configretriever/configretriever.cpp b/config/src/tests/configretriever/configretriever.cpp index 2c1954bcd95..d4ebe2da994 100644 --- a/config/src/tests/configretriever/configretriever.cpp +++ b/config/src/tests/configretriever/configretriever.cpp @@ -7,7 +7,9 @@ #include <vespa/config/retriever/simpleconfigretriever.h> #include <vespa/config/retriever/simpleconfigurer.h> #include <vespa/config/common/configholder.h> +#include <vespa/config/common/configcontext.h> #include <vespa/config/subscription/configsubscription.h> +#include <vespa/config/subscription/sourcespec.h> #include <vespa/config/common/exceptions.h> #include "config-bootstrap.h" #include "config-foo.h" @@ -39,7 +41,7 @@ struct ConfigTestFixture { bootstrapBuilder(), componentConfig(), set(), - context(new ConfigContext(set)), + context(std::make_shared<ConfigContext>(set)), idcounter(-1) { set.addBuilder(configId, &bootstrapBuilder); diff --git a/config/src/tests/configuri/configuri_test.cpp b/config/src/tests/configuri/configuri_test.cpp index c813761e763..38d49848596 100644 --- a/config/src/tests/configuri/configuri_test.cpp +++ b/config/src/tests/configuri/configuri_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> +#include <vespa/config/common/configcontext.h> #include "config-my.h" using namespace config; diff --git a/config/src/tests/failover/failover.cpp b/config/src/tests/failover/failover.cpp index 0ca09b228f3..99b6967c929 100644 --- a/config/src/tests/failover/failover.cpp +++ b/config/src/tests/failover/failover.cpp @@ -4,10 +4,12 @@ #include <vespa/config/common/misc.h> #include <vespa/config/frt/protocol.h> #include <vespa/config/config.h> +#include <vespa/config/common/configcontext.h> #include <vespa/fnet/frt/frt.h> #include "config-my.h" #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/simple_buffer.h> + #include <vespa/log/log.h> LOG_SETUP("failover"); @@ -189,7 +191,7 @@ struct ConfigCheckFixture { NetworkFixture & nf; ConfigCheckFixture(NetworkFixture & f2) - : ctx(new ConfigContext(testTimingValues, f2.spec)), + : ctx(std::make_shared<ConfigContext>(testTimingValues, f2.spec)), nf(f2) { } @@ -223,7 +225,7 @@ struct ConfigReloadFixture { ConfigHandle<MyConfig>::UP handle; ConfigReloadFixture(NetworkFixture & f2) - : ctx(new ConfigContext(testTimingValues, f2.spec)), + : ctx(std::make_shared<ConfigContext>(testTimingValues, f2.spec)), nf(f2), s(ctx), handle(s.subscribe<MyConfig>("myId")) diff --git a/config/src/tests/file_subscription/file_subscription.cpp b/config/src/tests/file_subscription/file_subscription.cpp index ceaf16c9191..468180cd78f 100644 --- a/config/src/tests/file_subscription/file_subscription.cpp +++ b/config/src/tests/file_subscription/file_subscription.cpp @@ -4,7 +4,8 @@ #include <vespa/config/common/configholder.h> #include <vespa/config/file/filesource.h> #include <vespa/config/common/exceptions.h> -#include <vespa/vespalib/util/sync.h> +#include <vespa/config/common/sourcefactory.h> +#include <vespa/config/common/configcontext.h> #include <fstream> #include <config-my.h> #include <config-foo.h> @@ -61,7 +62,7 @@ TEST("requireThatFileSpecGivesCorrectSource") { SourceFactory::UP factory(spec.createSourceFactory(TimingValues())); ASSERT_TRUE(factory); - IConfigHolder::SP holder(new ConfigHolder()); + auto holder = std::make_shared<ConfigHolder>(); Source::UP src = factory->createSource(holder, ConfigKey("my", "my", "bar", "foo")); ASSERT_TRUE(src); @@ -89,7 +90,7 @@ TEST("requireThatFileSubscriptionReturnsCorrectConfig") { TEST("requireThatReconfigIsCalledWhenConfigChanges") { writeFile("my.cfg", "foo"); { - IConfigContext::SP context(new ConfigContext(FileSpec("my.cfg"))); + auto context = std::make_shared<ConfigContext>(FileSpec("my.cfg")); ConfigSubscriber s(context); std::unique_ptr<ConfigHandle<MyConfig> > handle = s.subscribe<MyConfig>(""); s.nextConfigNow(); diff --git a/config/src/tests/getconfig/getconfig.cpp b/config/src/tests/getconfig/getconfig.cpp index 0cbac7aa456..b091e347ff2 100644 --- a/config/src/tests/getconfig/getconfig.cpp +++ b/config/src/tests/getconfig/getconfig.cpp @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/config/config.h> #include <vespa/config/helper/configgetter.hpp> -#include <vespa/config/raw/rawsource.h> +#include <vespa/config/common/configcontext.h> #include "config-my.h" using namespace config; @@ -16,7 +15,7 @@ struct ConfigFixture { ConfigContext::SP context; ConfigFixture() : builder(), set(), context() { set.addBuilder("cfgid", &builder); - context.reset(new ConfigContext(set)); + context = std::make_shared<ConfigContext>(set); } }; @@ -27,7 +26,7 @@ TEST("requireThatGetConfigReturnsCorrectConfig") RawSpec spec("myField \"foo\"\n"); std::unique_ptr<MyConfig> cfg = ConfigGetter<MyConfig>::getConfig("myid", spec); - ASSERT_TRUE(cfg.get() != NULL); + ASSERT_TRUE(cfg); ASSERT_EQUAL("my", cfg->defName()); ASSERT_EQUAL("foo", cfg->myField); } @@ -37,7 +36,7 @@ TEST("requireThatGetConfigReturnsCorrectConfig") { FileSpec spec(TEST_PATH("my.cfg")); std::unique_ptr<MyConfig> cfg = ConfigGetter<MyConfig>::getConfig("", spec); - ASSERT_TRUE(cfg.get() != NULL); + ASSERT_TRUE(cfg); ASSERT_EQUAL("my", cfg->defName()); ASSERT_EQUAL("foobar", cfg->myField); } diff --git a/config/src/tests/raw_subscription/raw_subscription.cpp b/config/src/tests/raw_subscription/raw_subscription.cpp index 478b4611bac..39062f4e7ac 100644 --- a/config/src/tests/raw_subscription/raw_subscription.cpp +++ b/config/src/tests/raw_subscription/raw_subscription.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include <vespa/config/common/configholder.h> +#include <vespa/config/common/sourcefactory.h> #include <vespa/config/raw/rawsource.h> #include "config-my.h" @@ -10,7 +11,7 @@ using namespace config; TEST("require that raw spec can create source factory") { RawSpec spec("myField \"foo\"\n"); - SourceFactory::UP raw = spec.createSourceFactory(TimingValues()); + auto raw = spec.createSourceFactory(TimingValues()); ASSERT_TRUE(raw); IConfigHolder::SP holder(new ConfigHolder()); Source::UP src = raw->createSource(holder, ConfigKey("myid", "my", "bar", "foo")); diff --git a/config/src/tests/unittest/unittest.cpp b/config/src/tests/unittest/unittest.cpp index 4b46dcef900..46aefe152af 100644 --- a/config/src/tests/unittest/unittest.cpp +++ b/config/src/tests/unittest/unittest.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> +#include <vespa/config/common/configcontext.h> #include "config-my.h" #include "config-foo.h" #include "config-bar.h" @@ -15,13 +16,13 @@ using namespace std::chrono_literals; namespace { void verifyConfig(const std::string & expected, std::unique_ptr<FooConfig> cfg) { - ASSERT_TRUE(cfg.get() != NULL); + ASSERT_TRUE(cfg); ASSERT_EQUAL(expected, cfg->fooValue); } void verifyConfig(const std::string & expected, std::unique_ptr<BarConfig> cfg) { - ASSERT_TRUE(cfg.get() != NULL); + ASSERT_TRUE(cfg); ASSERT_EQUAL(expected, cfg->barValue); } } @@ -37,7 +38,7 @@ TEST("requireThatUnitTestsCanBeCreated") { TEST("requireThatConfigCanBeReloaded") { ConfigSet set; - ConfigContext::SP ctx(new ConfigContext(set)); + auto ctx = std::make_shared<ConfigContext>(set); MyConfigBuilder builder; builder.myField = "myfoo"; set.addBuilder("myid", &builder); @@ -46,7 +47,7 @@ TEST("requireThatConfigCanBeReloaded") { ConfigHandle<MyConfig>::UP handle = subscriber.subscribe<MyConfig>("myid"); ASSERT_TRUE(subscriber.nextConfigNow()); std::unique_ptr<MyConfig> cfg(handle->getConfig()); - ASSERT_TRUE(cfg.get() != NULL); + ASSERT_TRUE(cfg); ASSERT_EQUAL("myfoo", cfg->myField); ctx->reload(); ASSERT_FALSE(subscriber.nextConfig(1000ms)); @@ -54,13 +55,13 @@ TEST("requireThatConfigCanBeReloaded") { ctx->reload(); ASSERT_TRUE(subscriber.nextConfig(10000ms)); cfg = handle->getConfig(); - ASSERT_TRUE(cfg.get() != NULL); + ASSERT_TRUE(cfg); ASSERT_EQUAL("foobar", cfg->myField); } TEST("requireThatCanSubscribeWithSameIdToDifferentDefs") { ConfigSet set; - ConfigContext::SP ctx(new ConfigContext(set)); + auto ctx = std::make_shared<ConfigContext>(set); FooConfigBuilder fooBuilder; BarConfigBuilder barBuilder; diff --git a/config/src/vespa/config/common/configcontext.cpp b/config/src/vespa/config/common/configcontext.cpp index ed549f0cb6a..0eed91296a6 100644 --- a/config/src/vespa/config/common/configcontext.cpp +++ b/config/src/vespa/config/common/configcontext.cpp @@ -8,26 +8,26 @@ namespace config { ConfigContext::ConfigContext(const SourceSpec & spec) : _timingValues(), _generation(1), - _manager(spec.createSourceFactory(_timingValues), _generation) + _manager(std::make_unique<ConfigManager>(spec.createSourceFactory(_timingValues), _generation)) { } ConfigContext::ConfigContext(const TimingValues & timingValues, const SourceSpec & spec) : _timingValues(timingValues), _generation(1), - _manager(spec.createSourceFactory(_timingValues), _generation) + _manager(std::make_unique<ConfigManager>(spec.createSourceFactory(_timingValues), _generation)) { } IConfigManager & ConfigContext::getManagerInstance() { - return _manager; + return *_manager; } void ConfigContext::reload() { _generation++; - _manager.reload(_generation); + _manager->reload(_generation); } } // namespace config diff --git a/config/src/vespa/config/common/configcontext.h b/config/src/vespa/config/common/configcontext.h index 7d9836fe70b..dd3296a9b01 100644 --- a/config/src/vespa/config/common/configcontext.h +++ b/config/src/vespa/config/common/configcontext.h @@ -3,7 +3,7 @@ #include "iconfigcontext.h" #include "timingvalues.h" -#include "configmanager.h" +#include "iconfigmanager.h" #include <vespa/config/subscription/sourcespec.h> namespace config { @@ -19,7 +19,7 @@ public: private: TimingValues _timingValues; int64_t _generation; - ConfigManager _manager; + std::unique_ptr<IConfigManager> _manager; }; diff --git a/config/src/vespa/config/common/configholder.cpp b/config/src/vespa/config/common/configholder.cpp index c2839b4a123..66c214f470b 100644 --- a/config/src/vespa/config/common/configholder.cpp +++ b/config/src/vespa/config/common/configholder.cpp @@ -5,7 +5,8 @@ namespace config { ConfigHolder::ConfigHolder() - : _monitor(), + : _lock(), + _cond(), _current() { } @@ -15,40 +16,39 @@ ConfigHolder::~ConfigHolder() = default; ConfigUpdate::UP ConfigHolder::provide() { - vespalib::MonitorGuard guard(_monitor); + std::lock_guard guard(_lock); return std::move(_current); } void ConfigHolder::handle(ConfigUpdate::UP update) { - vespalib::MonitorGuard guard(_monitor); + std::lock_guard guard(_lock); if (_current) { update->merge(*_current); } _current = std::move(update); - guard.broadcast(); + _cond.notify_all(); } bool ConfigHolder::wait(milliseconds timeoutInMillis) { - vespalib::MonitorGuard guard(_monitor); - return static_cast<bool>(_current) || guard.wait(timeoutInMillis); + std::unique_lock guard(_lock); + return static_cast<bool>(_current) || (_cond.wait_for(guard, timeoutInMillis) == std::cv_status::no_timeout); } bool ConfigHolder::poll() { - vespalib::MonitorGuard guard(_monitor); + std::lock_guard guard(_lock); return static_cast<bool>(_current); } void ConfigHolder::interrupt() { - vespalib::MonitorGuard guard(_monitor); - guard.broadcast(); + _cond.notify_all(); } } // namespace config diff --git a/config/src/vespa/config/common/configholder.h b/config/src/vespa/config/common/configholder.h index be65d6cff39..35bb6a8ccc2 100644 --- a/config/src/vespa/config/common/configholder.h +++ b/config/src/vespa/config/common/configholder.h @@ -2,7 +2,8 @@ #pragma once #include "iconfigholder.h" -#include <vespa/vespalib/util/sync.h> +#include <mutex> +#include <condition_variable> namespace config { @@ -21,8 +22,9 @@ public: bool poll() override; void interrupt() override; public: - vespalib::Monitor _monitor; - ConfigUpdate::UP _current; + std::mutex _lock; + std::condition_variable _cond; + ConfigUpdate::UP _current; }; } // namespace config diff --git a/config/src/vespa/config/common/configmanager.cpp b/config/src/vespa/config/common/configmanager.cpp index 11fe66a1c64..df8615c9c40 100644 --- a/config/src/vespa/config/common/configmanager.cpp +++ b/config/src/vespa/config/common/configmanager.cpp @@ -18,8 +18,7 @@ ConfigManager::ConfigManager(SourceFactory::UP sourceFactory, int64_t initialGen _sourceFactory(std::move(sourceFactory)), _generation(initialGeneration), _subscriptionMap(), - _lock(), - _firstLock() + _lock() { } ConfigManager::~ConfigManager() = default; @@ -50,7 +49,7 @@ ConfigManager::subscribe(const ConfigKey & key, milliseconds timeoutInMillis) throw ConfigTimeoutException(oss.str()); } LOG(debug, "done subscribing"); - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _subscriptionMap[id] = subscription; return subscription; } @@ -58,7 +57,7 @@ ConfigManager::subscribe(const ConfigKey & key, milliseconds timeoutInMillis) void ConfigManager::unsubscribe(const ConfigSubscription::SP & subscription) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); const SubscriptionId id(subscription->getSubscriptionId()); if (_subscriptionMap.find(id) != _subscriptionMap.end()) _subscriptionMap.erase(id); @@ -68,9 +67,9 @@ void ConfigManager::reload(int64_t generation) { _generation = generation; - vespalib::LockGuard guard(_lock); - for (SubscriptionMap::iterator it(_subscriptionMap.begin()), mt(_subscriptionMap.end()); it != mt; it++) { - it->second->reload(_generation); + std::lock_guard guard(_lock); + for (auto & entry : _subscriptionMap) { + entry.second->reload(_generation); } } diff --git a/config/src/vespa/config/common/configmanager.h b/config/src/vespa/config/common/configmanager.h index 6210a03c0d1..0252c6aa572 100644 --- a/config/src/vespa/config/common/configmanager.h +++ b/config/src/vespa/config/common/configmanager.h @@ -4,8 +4,8 @@ #include <vespa/config/subscription/configsubscription.h> #include "iconfigmanager.h" #include "sourcefactory.h" -#include <vespa/vespalib/util/sync.h> #include <map> +#include <mutex> namespace config { @@ -22,7 +22,7 @@ class ConfigManager : public IConfigManager { public: ConfigManager(SourceFactory::UP sourceFactory, int64_t initialGeneration); - ~ConfigManager(); + ~ConfigManager() override; // Implements IConfigManager ConfigSubscription::SP subscribe(const ConfigKey & key, milliseconds timeoutInMillis) override; @@ -40,8 +40,7 @@ private: typedef std::map<SubscriptionId, ConfigSubscription::SP> SubscriptionMap; SubscriptionMap _subscriptionMap; - vespalib::Lock _lock; - vespalib::Lock _firstLock; + std::mutex _lock; }; } // namespace config diff --git a/config/src/vespa/config/frt/frtconnection.h b/config/src/vespa/config/frt/frtconnection.h index bde3e79b83c..9e6ef688607 100644 --- a/config/src/vespa/config/frt/frtconnection.h +++ b/config/src/vespa/config/frt/frtconnection.h @@ -2,7 +2,6 @@ #pragma once #include "connection.h" -#include <vespa/vespalib/util/sync.h> #include <vespa/config/common/timingvalues.h> #include <atomic> diff --git a/config/src/vespa/config/frt/frtsource.cpp b/config/src/vespa/config/frt/frtsource.cpp index 94acabc5926..bee5158ecb8 100644 --- a/config/src/vespa/config/frt/frtsource.cpp +++ b/config/src/vespa/config/frt/frtsource.cpp @@ -3,6 +3,7 @@ #include "frtconfigresponse.h" #include "frtsource.h" #include <vespa/vespalib/util/closuretask.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".config.frt.frtsource"); @@ -73,7 +74,7 @@ FRTSource::RequestDone(FRT_RPCRequest * request) LOG(debug, "request aborted, stopping"); return; } - assert(_currentRequest.get() != NULL); + assert(_currentRequest); // If this was error from FRT side and nothing to do with config, notify // connection about the error. if (request->IsError()) { @@ -88,7 +89,7 @@ void FRTSource::close() { { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (_closed) return; LOG(spam, "Killing task"); @@ -106,7 +107,7 @@ FRTSource::close() void FRTSource::scheduleNextGetConfig() { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (_closed) return; double sec = _agent->getWaitTime() / 1000.0; diff --git a/config/src/vespa/config/frt/frtsource.h b/config/src/vespa/config/frt/frtsource.h index 16e1bfac37a..eabac3bf012 100644 --- a/config/src/vespa/config/frt/frtsource.h +++ b/config/src/vespa/config/frt/frtsource.h @@ -9,8 +9,6 @@ #include <vespa/config/common/source.h> #include <vespa/fnet/frt/invoker.h> -#include <vespa/vespalib/util/sync.h> - namespace config { /** @@ -21,7 +19,7 @@ class FRTSource : public Source, { public: FRTSource(const ConnectionFactory::SP & connectionFactory, const FRTConfigRequestFactory & requestFactory, ConfigAgent::UP agent, const ConfigKey & key); - ~FRTSource(); + ~FRTSource() override; void RequestDone(FRT_RPCRequest * request) override; void close() override; @@ -40,7 +38,7 @@ private: const ConfigKey _key; std::unique_ptr<FNET_Task> _task; - vespalib::Lock _lock; // Protects _task and _closed + std::mutex _lock; // Protects _task and _closed bool _closed; }; diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp index 4ad55726863..cfd248e5d86 100644 --- a/config/src/vespa/config/frt/protocol.cpp +++ b/config/src/vespa/config/frt/protocol.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> #include <sstream> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".config.frt.protocol"); diff --git a/config/src/vespa/config/helper/configfetcher.cpp b/config/src/vespa/config/helper/configfetcher.cpp index b013c137c2c..dda9c67f056 100644 --- a/config/src/vespa/config/helper/configfetcher.cpp +++ b/config/src/vespa/config/helper/configfetcher.cpp @@ -2,6 +2,7 @@ #include "configfetcher.h" #include <vespa/config/common/exceptions.h> +#include <vespa/config/common/configcontext.h> #include <vespa/vespalib/util/thread.h> #include <vespa/log/log.h> LOG_SETUP(".config.helper.configfetcher"); diff --git a/config/src/vespa/config/helper/legacysubscriber.hpp b/config/src/vespa/config/helper/legacysubscriber.hpp index bffe1d3feca..4aa383c6f9b 100644 --- a/config/src/vespa/config/helper/legacysubscriber.hpp +++ b/config/src/vespa/config/helper/legacysubscriber.hpp @@ -1,5 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/config/common/configcontext.h> + namespace config { template <typename ConfigType> @@ -9,10 +11,10 @@ LegacySubscriber::subscribe(const std::string & configId, IFetcherCallback<Confi if (isLegacyConfigId(configId)) { std::string legacyId(legacyConfigId2ConfigId(configId)); std::unique_ptr<SourceSpec> spec(legacyConfigId2Spec(configId)); - _fetcher.reset(new ConfigFetcher(IConfigContext::SP(new ConfigContext(*spec)))); + _fetcher = std::make_unique<ConfigFetcher>(std::make_shared<ConfigContext>(*spec)); _fetcher->subscribe<ConfigType>(legacyId, callback); } else { - _fetcher.reset(new ConfigFetcher()); + _fetcher = std::make_unique<ConfigFetcher>(); _fetcher->subscribe<ConfigType>(configId, callback); } _configId = configId; diff --git a/config/src/vespa/config/retriever/configretriever.cpp b/config/src/vespa/config/retriever/configretriever.cpp index 08e7f15f0c5..a77c8114322 100644 --- a/config/src/vespa/config/retriever/configretriever.cpp +++ b/config/src/vespa/config/retriever/configretriever.cpp @@ -2,6 +2,8 @@ #include "configretriever.h" #include <vespa/config/common/exceptions.h> +#include <vespa/config/subscription/sourcespec.h> +#include <cassert> using std::chrono::milliseconds; @@ -51,7 +53,7 @@ ConfigRetriever::getConfigs(const ConfigKeySet & keySet, milliseconds timeoutInM if (keySet != _lastKeySet) { _lastKeySet = keySet; { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (_closed) return ConfigSnapshot(); _configSubscriber = std::make_unique<GenericConfigSubscriber>(_context); @@ -82,7 +84,7 @@ ConfigRetriever::getConfigs(const ConfigKeySet & keySet, milliseconds timeoutInM void ConfigRetriever::close() { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _closed = true; _bootstrapSubscriber.close(); if (_configSubscriber) diff --git a/config/src/vespa/config/retriever/configretriever.h b/config/src/vespa/config/retriever/configretriever.h index 14e3171fc62..1207a17af97 100644 --- a/config/src/vespa/config/retriever/configretriever.h +++ b/config/src/vespa/config/retriever/configretriever.h @@ -1,17 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "configkeyset.h" #include "configsnapshot.h" #include "genericconfigsubscriber.h" #include "fixedconfigsubscriber.h" #include <vespa/config/common/configkey.h> #include <vespa/config/subscription/configsubscription.h> #include <vespa/vespalib/stllike/string.h> - +#include <mutex> namespace config { +class SourceSpec; + /** * A ConfigRetriever is a helper class for retrieving a set of dynamically * changing and depending configs. You should use this class whenever you have a @@ -94,7 +95,7 @@ public: private: FixedConfigSubscriber _bootstrapSubscriber; std::unique_ptr<GenericConfigSubscriber> _configSubscriber; - vespalib::Lock _lock; + std::mutex _lock; std::vector<ConfigSubscription::SP> _subscriptionList; ConfigKeySet _lastKeySet; IConfigContext::SP _context; diff --git a/config/src/vespa/config/retriever/simpleconfigurer.cpp b/config/src/vespa/config/retriever/simpleconfigurer.cpp index 1f8868f9d6a..0091fa7bfd5 100644 --- a/config/src/vespa/config/retriever/simpleconfigurer.cpp +++ b/config/src/vespa/config/retriever/simpleconfigurer.cpp @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "simpleconfigurer.h" +#include <cassert> + #include <vespa/log/log.h> LOG_SETUP(".config.retriever.simpleconfigurer"); -#include "simpleconfigurer.h" namespace config { @@ -12,7 +14,7 @@ SimpleConfigurer::SimpleConfigurer(SimpleConfigRetriever::UP retriever, SimpleCo _thread(*this), _started(false) { - assert(_retriever.get() != NULL); + assert(_retriever); } void diff --git a/config/src/vespa/config/subscription/configsubscriber.cpp b/config/src/vespa/config/subscription/configsubscriber.cpp index b226c149b9e..cb735e4353a 100644 --- a/config/src/vespa/config/subscription/configsubscriber.cpp +++ b/config/src/vespa/config/subscription/configsubscriber.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configsubscriber.h" +#include <vespa/config/common/configcontext.h> namespace config { diff --git a/config/src/vespa/config/subscription/configsubscriber.h b/config/src/vespa/config/subscription/configsubscriber.h index 4787c6cd858..d7139b5d61f 100644 --- a/config/src/vespa/config/subscription/configsubscriber.h +++ b/config/src/vespa/config/subscription/configsubscriber.h @@ -7,6 +7,7 @@ #include "configsubscriptionset.h" #include "configprovider.h" #include "sourcespec.h" +#include <vespa/config/common/timingvalues.h> namespace config { diff --git a/config/src/vespa/config/subscription/configsubscription.h b/config/src/vespa/config/subscription/configsubscription.h index 80051edb86b..a48fddaa41c 100644 --- a/config/src/vespa/config/subscription/configsubscription.h +++ b/config/src/vespa/config/subscription/configsubscription.h @@ -1,10 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <memory> +#include "subscriptionid.h" #include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/configkey.h> #include <vespa/config/common/source.h> -#include "subscriptionid.h" #include <atomic> #include <chrono> diff --git a/config/src/vespa/config/subscription/configsubscriptionset.h b/config/src/vespa/config/subscription/configsubscriptionset.h index 181f79f0ef2..3e91f1cc68d 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.h +++ b/config/src/vespa/config/subscription/configsubscriptionset.h @@ -2,13 +2,12 @@ // #pragma once -#include <vespa/config/common/iconfigholder.h> -#include <vespa/config/common/configcontext.h> #include "confighandle.h" #include "subscriptionid.h" #include "configsubscription.h" #include "configprovider.h" - +#include <vespa/config/common/iconfigcontext.h> +#include <vespa/config/common/iconfigmanager.h> #include <atomic> namespace config { diff --git a/config/src/vespa/config/subscription/sourcespec.cpp b/config/src/vespa/config/subscription/sourcespec.cpp index 30926d75465..326b3191fd0 100644 --- a/config/src/vespa/config/subscription/sourcespec.cpp +++ b/config/src/vespa/config/subscription/sourcespec.cpp @@ -10,6 +10,7 @@ #include <vespa/config/set/configinstancesourcefactory.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/config/print/asciiconfigwriter.h> +#include <cassert> namespace config { diff --git a/config/src/vespa/config/subscription/subscriptionid.h b/config/src/vespa/config/subscription/subscriptionid.h index fe87debb58f..67dd905f523 100644 --- a/config/src/vespa/config/subscription/subscriptionid.h +++ b/config/src/vespa/config/subscription/subscriptionid.h @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <cstdint> + namespace config { typedef uint64_t SubscriptionId; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 4fba0204f55..083112d8456 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -132,6 +132,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final TesterClient testerClient; private final Metric metric; private final BooleanFlag deployWithInternalRestart; + private final BooleanFlag acquireProvisionLock; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -182,6 +183,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.testerClient = Objects.requireNonNull(testerClient); this.metric = Objects.requireNonNull(metric); this.deployWithInternalRestart = Flags.DEPLOY_WITH_INTERNAL_RESTART.bindTo(flagSource); + this.acquireProvisionLock = Flags.ALWAYS_ACQUIRE_PROVISION_LOCK.bindTo(flagSource); } public static class Builder { @@ -508,7 +510,15 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // and allocated hosts in model and handlers in RPC server transaction.add(tenantApplications.createDeleteTransaction(applicationId)); - hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); + hostProvisioner.ifPresent(provisioner -> { + if (acquireProvisionLock.value()) { + try (var provisionLock = provisioner.lock(applicationId)) { + provisioner.remove(transaction, provisionLock); + } + } else { + provisioner.remove(transaction, applicationId); + } + }); transaction.onCommitted(() -> log.log(Level.INFO, "Deleted " + applicationId)); transaction.commit(); return true; @@ -722,13 +732,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Session operations ---------------------------------------------------------------- - - public CompletionWaiter activate(LocalSession session, Session previousActiveSession, ApplicationId applicationId, boolean force) { CompletionWaiter waiter = session.getSessionZooKeeperClient().createActiveWaiter(); NestedTransaction transaction = new NestedTransaction(); transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, force)); - hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); + hostProvisioner.ifPresent(provisioner -> { + if (acquireProvisionLock.value()) { + try (var lock = provisioner.lock(applicationId)) { + provisioner.activate(transaction, session.getAllocatedHosts().getHosts(), lock); + } + } else { + provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts()); + } + }); transaction.commit(); return waiter; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java index 8e7e4eec9b0..a185bcaab8a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.metrics; import ai.vespa.util.http.VespaHttpClientBuilder; -import java.util.logging.Level; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -13,13 +13,18 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import java.io.IOException; import java.net.URI; +import java.time.Duration; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** @@ -39,13 +44,15 @@ public class ClusterDeploymentMetricsRetriever { private static final List<String> WANTED_METRIC_SERVICES = List.of(VESPA_CONTAINER, VESPA_QRSERVER, VESPA_DISTRIBUTOR); + private static final ExecutorService executor = Executors.newFixedThreadPool(10, new DaemonThreadFactory("cluster-deployment-metrics-retriever-")); private static final CloseableHttpClient httpClient = VespaHttpClientBuilder .create(PoolingHttpClientConnectionManager::new) .setDefaultRequestConfig( RequestConfig.custom() - .setConnectTimeout(10 * 1000) - .setSocketTimeout(10 * 1000) + .setConnectionRequestTimeout((int)Duration.ofSeconds(60).toMillis()) + .setConnectTimeout((int)Duration.ofSeconds(10).toMillis()) + .setSocketTimeout((int)Duration.ofSeconds(10).toMillis()) .build()) .build(); @@ -57,19 +64,20 @@ public class ClusterDeploymentMetricsRetriever { Map<ClusterInfo, DeploymentMetricsAggregator> clusterMetricsMap = new ConcurrentHashMap<>(); long startTime = System.currentTimeMillis(); - Runnable retrieveMetricsJob = () -> - hosts.parallelStream().forEach(host -> - getHostMetrics(host, clusterMetricsMap) - ); - - ForkJoinPool threadPool = new ForkJoinPool(10); - threadPool.submit(retrieveMetricsJob); - threadPool.shutdown(); - + List<Callable<Void>> jobs = hosts.stream() + .map(hostUri -> (Callable<Void>) () -> { + try { + getHostMetrics(hostUri, clusterMetricsMap); + } catch (Exception e) { + log.log(Level.FINE, e, () -> "Failed to download metrics: " + e.getMessage()); + } + return null; + }) + .collect(Collectors.toList()); try { - threadPool.awaitTermination(1, TimeUnit.MINUTES); + executor.invokeAll(jobs, 1, TimeUnit.SECONDS); } catch (InterruptedException e) { - throw new RuntimeException(e); + throw new RuntimeException("Failed to retrieve metrics in time: " + e.getMessage(), e); } log.log(Level.FINE, () -> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java index ead1e79a416..fe3155b251c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.configchange; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import java.util.ArrayList; @@ -27,9 +28,10 @@ public class ConfigChangeActionsBuilder { } public ConfigChangeActionsBuilder restart(String message, String clusterName, String clusterType, String serviceType, String serviceName, boolean ignoreForInternalRedeploy) { - actions.add(new VespaRestartAction(message, - createService(clusterName, clusterType, serviceType, serviceName), - ignoreForInternalRedeploy)); + actions.add(new VespaRestartAction(ClusterSpec.Id.from(clusterName), + message, + createService(clusterName, clusterType, serviceType, serviceName), + ignoreForInternalRedeploy)); return this; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 7553583e70c..87122fdba45 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -22,6 +22,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; @@ -227,15 +228,28 @@ public class DeployTester { } @Override + public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) { + } + + @Override public void remove(NestedTransaction transaction, ApplicationId application) { // noop } @Override + public void remove(NestedTransaction transaction, ProvisionLock lock) { + } + + @Override public void restart(ApplicationId application, HostFilter filter) { // noop } + @Override + public ProvisionLock lock(ApplicationId application) { + return null; + } + } private static class FailingModelFactory implements ModelFactory { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 18ca54e8c11..52fdb3e7c1f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -15,6 +15,7 @@ import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.HostedConfigModelRegistry; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -377,8 +378,10 @@ public class HostedDeployTest { new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName")); List<ModelFactory> modelFactories = List.of( - new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), new VespaRestartAction("change", services)), - new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), new VespaRestartAction("other change", services))); + new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), + new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services)), + new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), + new VespaRestartAction(ClusterSpec.Id.from("test"), "other change", services))); DeployTester tester = createTester(hosts, modelFactories, prodZone); PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.2.0"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 9d696ae34e7..32a9e867684 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.container.jdisc.HttpRequest; @@ -102,17 +103,32 @@ public class SessionHandlerTest { } @Override + public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) { + + } + + @Override public void remove(NestedTransaction transaction, ApplicationId application) { removed = true; lastApplicationId = application; } @Override + public void remove(NestedTransaction transaction, ProvisionLock lock) { + + } + + @Override public void restart(ApplicationId application, HostFilter filter) { restarted = true; lastApplicationId = application; } + @Override + public ProvisionLock lock(ApplicationId application) { + return null; + } + } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index 712242a69e6..d092a3139a0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.transaction.NestedTransaction; @@ -95,15 +96,30 @@ class MaintainerTester { } @Override + public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) { + + } + + @Override public void remove(NestedTransaction transaction, ApplicationId application) { // noop } @Override + public void remove(NestedTransaction transaction, ProvisionLock lock) { + + } + + @Override public void restart(ApplicationId application, HostFilter filter) { // noop } + @Override + public ProvisionLock lock(ApplicationId application) { + return null; + } + } -}
\ No newline at end of file +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 3858388ed39..43504988d67 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -17,6 +17,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; @@ -422,11 +423,26 @@ public class SessionPreparerTest { public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { } @Override + public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) { + + } + + @Override public void remove(NestedTransaction transaction, ApplicationId application) { } @Override + public void remove(NestedTransaction transaction, ProvisionLock lock) { + + } + + @Override public void restart(ApplicationId application, HostFilter filter) { } + @Override + public ProvisionLock lock(ApplicationId application) { + return null; + } + } } diff --git a/configutil/src/tests/config_status/config_status_test.cpp b/configutil/src/tests/config_status/config_status_test.cpp index 3cc5b2f4525..4d85b22a5c4 100644 --- a/configutil/src/tests/config_status/config_status_test.cpp +++ b/configutil/src/tests/config_status/config_status_test.cpp @@ -5,6 +5,7 @@ #include <vespa/config-model.h> #include <vespa/config/config.h> #include <vespa/config/subscription/sourcespec.h> +#include <vespa/config/common/configcontext.h> using namespace config; using vespalib::Portal; @@ -54,7 +55,7 @@ public: { flags.verbose = true; ConfigSet set; - ConfigContext::SP ctx(new ConfigContext(set)); + auto ctx = std::make_shared<ConfigContext>(set); cloud::config::ModelConfigBuilder builder; cloud::config::ModelConfigBuilder::Hosts::Services::Ports port; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 548ad0af484..8c21e6facec 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -166,7 +166,7 @@ enum Policy { /** Read the generated bills */ billingInformationRead(Privilege.grant(Action.read) .on(PathGroup.billingList) - .in(SystemName.PublicCd)), + .in(SystemName.PublicCd, SystemName.Public)), /** Invoice management */ hostedAccountant(Privilege.grant(Action.all()) diff --git a/dist/vespa.spec b/dist/vespa.spec index 71fd3170338..b41a4e420a0 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -110,7 +110,7 @@ BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %endif -BuildRequires: xxhash-devel >= 0.7.3 +BuildRequires: xxhash-devel >= 0.8.0 BuildRequires: openblas-devel BuildRequires: zlib-devel BuildRequires: re2-devel @@ -151,7 +151,7 @@ Requires: valgrind %endif Requires: Judy Requires: xxhash -Requires: xxhash-libs >= 0.7.3 +Requires: xxhash-libs >= 0.8.0 %if 0%{?el8} Requires: openblas %else @@ -252,7 +252,7 @@ Vespa - The open big data serving engine - base Summary: Vespa - The open big data serving engine - base C++ libs -Requires: xxhash-libs >= 0.7.3 +Requires: xxhash-libs >= 0.8.0 %if 0%{?el7} Requires: vespa-openssl >= 1.1.1g-1 %else diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerEngine.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerEngine.java index 6c831cef7d7..1f2a35a2a38 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerEngine.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerEngine.java @@ -44,6 +44,7 @@ import java.util.logging.Logger; import java.util.stream.Stream; public class DockerEngine implements ContainerEngine { + private static final Logger logger = Logger.getLogger(DockerEngine.class.getName()); static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby"; diff --git a/document/src/vespa/document/base/field.h b/document/src/vespa/document/base/field.h index 613fcd9df1b..3a0c9b82136 100644 --- a/document/src/vespa/document/base/field.h +++ b/document/src/vespa/document/base/field.h @@ -97,13 +97,13 @@ public: const DataType &getDataType() const { return *_dataType; } - int getId() const { return _fieldId; } + int getId() const noexcept { return _fieldId; } vespalib::string toString(bool verbose=false) const; bool contains(const FieldSet& fields) const override; Type getType() const override { return Type::FIELD; } bool valid() const { return _fieldId != 0; } - uint32_t hash() const { return getId(); } + uint32_t hash() const noexcept { return getId(); } private: int calculateIdV7(); diff --git a/document/src/vespa/document/bucket/CMakeLists.txt b/document/src/vespa/document/bucket/CMakeLists.txt index 7704fe94bd0..fbbfe8b8466 100644 --- a/document/src/vespa/document/bucket/CMakeLists.txt +++ b/document/src/vespa/document/bucket/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_add_library(document_bucket OBJECT SOURCES bucket.cpp - bucketdistribution.cpp bucketid.cpp bucketidfactory.cpp bucketidlist.cpp diff --git a/document/src/vespa/document/bucket/bucketdistribution.cpp b/document/src/vespa/document/bucket/bucketdistribution.cpp deleted file mode 100644 index 34ce9a9eccf..00000000000 --- a/document/src/vespa/document/bucket/bucketdistribution.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketdistribution.h" -#include <vespa/log/log.h> - -LOG_SETUP(".bucketdistribution"); - -namespace document { - -BucketDistribution::BucketDistribution(uint32_t numColumns, uint32_t numBucketBits) : - _numColumns(0), - _numBucketBits(numBucketBits), - _bucketToColumn(), - _lock() -{ - _bucketToColumn.resize(getNumBuckets()); - reset(); - setNumColumns(numColumns); -} - -void -BucketDistribution::getBucketCount(uint32_t numColumns, uint32_t numBucketBits, std::vector<uint32_t> &ret) -{ - ret.resize(numColumns); - uint32_t cnt = getNumBuckets(numBucketBits) / numColumns; - uint32_t rst = getNumBuckets(numBucketBits) % numColumns; - for (uint32_t i = 0; i < numColumns; ++i) { - ret[i] = cnt + (i < rst ? 1 : 0); - } -} - -void -BucketDistribution::getBucketMigrateCount(uint32_t numColumns, uint32_t numBucketBits, std::vector<uint32_t> &ret) -{ - getBucketCount(numColumns++, numBucketBits, ret); - uint32_t cnt = getNumBuckets(numBucketBits) / numColumns; - uint32_t rst = getNumBuckets(numBucketBits) % numColumns; - for (uint32_t i = 0; i < numColumns - 1; ++i) { - ret[i] -= cnt + (i < rst ? 1 : 0); - } -} - -void -BucketDistribution::reset() -{ - for (std::vector<uint32_t>::iterator it = _bucketToColumn.begin(); - it != _bucketToColumn.end(); ++it) { - *it = 0; - } - _numColumns = 1; -} - -void -BucketDistribution::addColumn() -{ - uint32_t newColumns = _numColumns + 1; - std::vector<uint32_t> migrate; - getBucketMigrateCount(_numColumns, _numBucketBits, migrate); - uint32_t numBuckets = getNumBuckets(_numBucketBits); - for (uint32_t i = 0; i < numBuckets; ++i) { - uint32_t old = _bucketToColumn[i]; - if (migrate[old] > 0) { - _bucketToColumn[i] = _numColumns; // move this bucket to the new column - migrate[old]--; - } - } - _numColumns = newColumns; -} - -void -BucketDistribution::setNumColumns(uint32_t numColumns) -{ - vespalib::LockGuard guard(_lock); - if (numColumns < _numColumns) { - reset(); - } - if (numColumns == _numColumns) { - return; - } - for (int i = numColumns - _numColumns; --i >= 0; ) { - addColumn(); - } -} - -void -BucketDistribution::setNumBucketBits(uint32_t numBucketBits) -{ - uint32_t numColumns; - { - vespalib::LockGuard guard(_lock); - if (numBucketBits == _numBucketBits) { - return; - } - _numBucketBits = numBucketBits; - _bucketToColumn.resize(getNumBuckets(numBucketBits)); - numColumns = _numColumns; - reset(); - } - setNumColumns(numColumns); -} - -uint32_t -BucketDistribution::getColumn(const document::BucketId &bucketId) const -{ - uint32_t ret = (uint32_t)(bucketId.getId() & (getNumBuckets(_numBucketBits) - 1)); - if (ret >= _bucketToColumn.size()) { - LOG(error, - "The bucket distribution map is not in sync with the number of bucket bits. " - "This should never happen! Distribution is broken!!"); - return 0; - } - return _bucketToColumn[ret]; -} - -} diff --git a/document/src/vespa/document/bucket/bucketdistribution.h b/document/src/vespa/document/bucket/bucketdistribution.h deleted file mode 100644 index c451ed9b9a7..00000000000 --- a/document/src/vespa/document/bucket/bucketdistribution.h +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class document::BucketDistribution - * \ingroup bucket - * - * Stable algorithmic hash distribution; this class assigns hash buckets to - * targets. The number of hash buckets should be large compared to the number - * of targets. The mapping from hash value to hash bucket is performed outside - * this class. - * - * This is used to determine which search column a bucket should go to. - */ -#pragma once - -#include "bucketid.h" -#include <vector> -#include <vespa/vespalib/util/sync.h> - -namespace document { - -class BucketDistribution { -public: - /** - * Constructs a new bucket distribution object with a given number of - * columns and buckets. - * - * @param numColumns The number of columns to distribute to. - * @param numBucketBits The number of bits to use for bucket id. - */ - BucketDistribution(uint32_t numColumns, uint32_t numBucketBits); - - /** - * Returns the number of buckets that the given number of bucket bits will - * allow. - * - * @param numBucketBits The number of bits to use for bucket id. - * @return The number of buckets allowed. - */ - static uint32_t getNumBuckets(uint32_t numBucketBits) { return 1 << numBucketBits; } - - /** - * This method returns a list that contains the distributions of the given - * number of buckets over the given number of columns. - * - * @param numColumns The number of columns to distribute to. - * @param numBucketBits The number of bits to use for bucket id. - * @param ret List to fill with the bucket distribution. - */ - static void getBucketCount(uint32_t numColumns, uint32_t numBucketBits, - std::vector<uint32_t> &ret); - - /** - * This method returns a list similar to getBucketCount(int,int), except - * that the returned list contains the number of buckets that will have to - * be migrated from each column if an additional column was added. - * - * @param numColumns The original number of columns. - * @param numBucketBits The number of bits to use for bucket id. - * @param ret List to fill with the number of buckets to migrate, - * one value per column. - */ - static void getBucketMigrateCount(uint32_t numColumns, - uint32_t numBucketBits, std::vector<uint32_t> &ret); - - /** - * Sets the number of columns to distribute to to 1, and resets the content - * of the internal bucket-to-column map so that it all buckets point to - * that single column. - */ - void reset(); - - /** - * Sets the number of columns to use for this document distribution object. - * This will reset and setup this object from scratch. The original number - * of buckets is maintained. - * - * @param numColumns The new number of columns to distribute to. - */ - void setNumColumns(uint32_t numColumns); - - /** - * Returns the number of columns to distribute to. - * - * @return The number of columns. - */ - uint32_t getNumColumns() const { return _numColumns; } - - /** - * Sets the number of buckets to use for this document distribution object. - * This will reset and setup this object from scratch. The original number - * of columns is maintained. - * - * @param numBucketBits The new number of bits to use for bucket id. - */ - void setNumBucketBits(uint32_t numBucketBits); - - /** - * Returns the number of bits used for bucket identifiers. - * - * @return The number of bits. - */ - uint32_t getNumBucketBits() const { return _numBucketBits; } - - /** - * Returns the number of buckets available using the configured number of - * bucket bits. - * - * @return The number of buckets. - */ - uint32_t getNumBuckets() const { return getNumBuckets(_numBucketBits); } - - /** - * This method maps the given bucket id to its corresponding column. - * - * @param bucketId The bucket whose column to lookup. - * @return The column to distribute the bucket to. - */ - uint32_t getColumn(const document::BucketId &bucketId) const; - -private: - /** - * Adds a single column to this bucket distribution object. This will - * modify the internal bucket-to-column map so that it takes into account - * the new column. - */ - void addColumn(); - -private: - uint32_t _numColumns; // The number of columns to distribute to. - uint32_t _numBucketBits; // The number of bits to use for bucket identification. - std::vector<uint32_t> _bucketToColumn; // A map from bucket id to column index. - vespalib::Lock _lock; -}; - -} // document - diff --git a/document/src/vespa/document/datatype/positiondatatype.cpp b/document/src/vespa/document/datatype/positiondatatype.cpp index a8e9c81c895..69e63d2ff72 100644 --- a/document/src/vespa/document/datatype/positiondatatype.cpp +++ b/document/src/vespa/document/datatype/positiondatatype.cpp @@ -10,9 +10,6 @@ const vespalib::string ZCURVE("_zcurve"); } -StructDataType::UP PositionDataType::_instance; -vespalib::Lock PositionDataType::_lock; - const vespalib::string PositionDataType::STRUCT_NAME("position"); const vespalib::string PositionDataType::FIELD_X("x"); const vespalib::string PositionDataType::FIELD_Y("y"); @@ -20,7 +17,7 @@ const vespalib::string PositionDataType::FIELD_Y("y"); StructDataType::UP PositionDataType::createInstance() { - StructDataType::UP type(new StructDataType(PositionDataType::STRUCT_NAME)); + auto type = std::make_unique<StructDataType>(PositionDataType::STRUCT_NAME); type->addField(Field(PositionDataType::FIELD_X, *DataType::INT)); type->addField(Field(PositionDataType::FIELD_Y, *DataType::INT)); return type; @@ -29,13 +26,8 @@ PositionDataType::createInstance() const StructDataType & PositionDataType::getInstance() { - if ( ! _instance) { - vespalib::LockGuard guard(_lock); - if ( ! _instance) { - _instance = createInstance(); - } - } - return *_instance; + static StructDataType::UP instance = createInstance(); + return *instance; } vespalib::string diff --git a/document/src/vespa/document/datatype/positiondatatype.h b/document/src/vespa/document/datatype/positiondatatype.h index fa8e44ad847..45ac35b1388 100644 --- a/document/src/vespa/document/datatype/positiondatatype.h +++ b/document/src/vespa/document/datatype/positiondatatype.h @@ -2,15 +2,11 @@ #pragma once #include <vespa/document/datatype/structdatatype.h> -#include <vespa/vespalib/util/sync.h> namespace document { class PositionDataType { private: - static StructDataType::UP _instance; - static vespalib::Lock _lock; - PositionDataType(); static StructDataType::UP createInstance(); diff --git a/document/src/vespa/document/datatype/urldatatype.cpp b/document/src/vespa/document/datatype/urldatatype.cpp index 00ea31408af..4db16f4705d 100644 --- a/document/src/vespa/document/datatype/urldatatype.cpp +++ b/document/src/vespa/document/datatype/urldatatype.cpp @@ -4,9 +4,6 @@ namespace document { -StructDataType::UP UrlDataType::_instance; -vespalib::Lock UrlDataType::_lock; - const vespalib::string UrlDataType::STRUCT_NAME("url"); const vespalib::string UrlDataType::FIELD_ALL("all"); const vespalib::string UrlDataType::FIELD_SCHEME("scheme"); @@ -19,7 +16,7 @@ const vespalib::string UrlDataType::FIELD_FRAGMENT("fragment"); StructDataType::UP UrlDataType::createInstance() { - StructDataType::UP type(new StructDataType(UrlDataType::STRUCT_NAME)); + auto type = std::make_unique<StructDataType>(UrlDataType::STRUCT_NAME); type->addField(Field(UrlDataType::FIELD_ALL, *DataType::STRING)); type->addField(Field(UrlDataType::FIELD_SCHEME, *DataType::STRING)); type->addField(Field(UrlDataType::FIELD_HOST, *DataType::STRING)); @@ -33,13 +30,8 @@ UrlDataType::createInstance() const StructDataType & UrlDataType::getInstance() { - if ( ! _instance ) { - vespalib::LockGuard guard(_lock); - if ( ! _instance ) { - _instance = createInstance(); - } - } - return *_instance; + static StructDataType::UP instance = createInstance(); + return *instance; } } // document diff --git a/document/src/vespa/document/datatype/urldatatype.h b/document/src/vespa/document/datatype/urldatatype.h index 902abd940e3..19ae6dde2e1 100644 --- a/document/src/vespa/document/datatype/urldatatype.h +++ b/document/src/vespa/document/datatype/urldatatype.h @@ -2,15 +2,11 @@ #pragma once #include <vespa/document/datatype/structdatatype.h> -#include <vespa/vespalib/util/sync.h> namespace document { class UrlDataType { private: - static StructDataType::UP _instance; - static vespalib::Lock _lock; - UrlDataType() { /* hide */ } static StructDataType::UP createInstance(); diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index bcd4edb3ab8..578d2999038 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -15,6 +15,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/config/config-documenttypes.h> #include <fstream> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".documenttyperepo"); @@ -547,13 +548,13 @@ DocumentTypeRepo::~DocumentTypeRepo() { } const DocumentType * -DocumentTypeRepo::getDocumentType(int32_t type_id) const { +DocumentTypeRepo::getDocumentType(int32_t type_id) const noexcept { const DataTypeRepo *repo = FindPtr(*_doc_types, type_id); return repo ? repo->doc_type : nullptr; } const DocumentType * -DocumentTypeRepo::getDocumentType(stringref name) const { +DocumentTypeRepo::getDocumentType(stringref name) const noexcept { DocumentTypeMap::const_iterator it = _doc_types->find(DocumentType::createId(name)); if (it != _doc_types->end() && it->second->doc_type->getName() == name) { diff --git a/document/src/vespa/document/repo/documenttyperepo.h b/document/src/vespa/document/repo/documenttyperepo.h index f0c59918a74..fd17bd5640a 100644 --- a/document/src/vespa/document/repo/documenttyperepo.h +++ b/document/src/vespa/document/repo/documenttyperepo.h @@ -34,8 +34,8 @@ public: explicit DocumentTypeRepo(const DocumenttypesConfig & config); ~DocumentTypeRepo(); - const DocumentType *getDocumentType(int32_t doc_type_id) const; - const DocumentType *getDocumentType(vespalib::stringref name) const; + const DocumentType *getDocumentType(int32_t doc_type_id) const noexcept; + const DocumentType *getDocumentType(vespalib::stringref name) const noexcept; const DataType *getDataType(const DocumentType &doc_type, int32_t id) const; const DataType *getDataType(const DocumentType &doc_type, vespalib::stringref name) const; const AnnotationType *getAnnotationType(const DocumentType &doc_type, int32_t id) const; diff --git a/document/src/vespa/document/repo/fixedtyperepo.cpp b/document/src/vespa/document/repo/fixedtyperepo.cpp index 81a26265830..20865a8a6ca 100644 --- a/document/src/vespa/document/repo/fixedtyperepo.cpp +++ b/document/src/vespa/document/repo/fixedtyperepo.cpp @@ -5,7 +5,7 @@ namespace document { -FixedTypeRepo::FixedTypeRepo(const DocumentTypeRepo &repo, const vespalib::string &type) +FixedTypeRepo::FixedTypeRepo(const DocumentTypeRepo &repo, const vespalib::string &type) noexcept : _repo(&repo), _doc_type(repo.getDocumentType(type)) { assert(_doc_type); diff --git a/document/src/vespa/document/repo/fixedtyperepo.h b/document/src/vespa/document/repo/fixedtyperepo.h index 67e7571e31d..29bef846e36 100644 --- a/document/src/vespa/document/repo/fixedtyperepo.h +++ b/document/src/vespa/document/repo/fixedtyperepo.h @@ -13,17 +13,17 @@ class FixedTypeRepo { const DocumentType *_doc_type; public: - explicit FixedTypeRepo(const DocumentTypeRepo &repo) + explicit FixedTypeRepo(const DocumentTypeRepo &repo) noexcept : _repo(&repo), _doc_type(repo.getDefaultDocType()) {} - FixedTypeRepo(const DocumentTypeRepo &repo, const DocumentType &doc_type) + FixedTypeRepo(const DocumentTypeRepo &repo, const DocumentType &doc_type) noexcept : _repo(&repo), _doc_type(&doc_type) {} - FixedTypeRepo(const DocumentTypeRepo &repo, const vespalib::string &type); + FixedTypeRepo(const DocumentTypeRepo &repo, const vespalib::string &type) noexcept; const DataType *getDataType(int32_t id) const { return _repo->getDataType(*_doc_type, id); } const DataType *getDataType(const vespalib::string &name) const { return _repo->getDataType(*_doc_type, name); } const AnnotationType *getAnnotationType(int32_t id) const { return _repo->getAnnotationType(*_doc_type, id); } const DocumentTypeRepo &getDocumentTypeRepo() const { return *_repo; } - const DocumentType &getDocumentType() const { return *_doc_type; } + const DocumentType &getDocumentType() const noexcept { return *_doc_type; } }; } // namespace document diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 73fc1c6486b..36cb92dfe33 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -2,7 +2,6 @@ #include "valuenodes.h" #include "visitor.h" #include "parser.h" -#include <vespa/document/bucket/bucketdistribution.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/fieldvalue/fieldvalues.h> diff --git a/document/src/vespa/document/util/queue.h b/document/src/vespa/document/util/queue.h index af0ce85fad1..7e3c98333c8 100644 --- a/document/src/vespa/document/util/queue.h +++ b/document/src/vespa/document/util/queue.h @@ -5,8 +5,7 @@ #include <vespa/vespalib/util/sync.h> #define UNUSED_PARAM(p) -namespace document -{ +namespace document { // XXX move to vespalib (or remove) /** @@ -202,79 +201,7 @@ public: } return retval; } -#if 0 -// XXX unused? - size_t max() const { return _max; } - size_t lowWaterMark() const { return _lowWaterMark; } - void max(size_t v) - { - vespalib::MonitorGuard guard(this->_cond); - _max = v; _lowWaterMark = _max/2; - } - void lowWaterMark(size_t v) - { - vespalib::MonitorGuard guard(this->_cond); - _lowWaterMark = v; - } -#endif -}; - -template <typename T, typename Q=std::queue<T> > -class QueueWithMaxSerialized : public QueueWithMax<T, Q> -{ -public: - QueueWithMaxSerialized(size_t max_=1000000, size_t lowWaterMark_=500000) : QueueWithMax<T, Q>(max_, lowWaterMark_) { } - virtual void add(const T& msg) - { - if (msg != NULL) { - this->_size += msg->getSerializedSize(); - } - } - virtual void sub(const T& msg) - { - if (msg != NULL) { - this->_size -= msg->getSerializedSize(); - } - } -}; - -#if 0 - -/** - This is an fast Q that reduces lock/unlock to a minimum and ditto with - context swithes on notify/wait. enque/deque have no atomic operations - unless it is empty/full. This limits the use to situations where there are - both single consumers and single producers. -*/ -template <typename T> -class QueueSingleProducerConsumer { -private: - typedef std::vector<T> Q; -public: - typedef typename Q::iterator iterator; - enum { end=-1 }; - QueueSingleProducerConsumer(size_t max=1000, - size_t highWaterMark=999, - size_t lowWaterMark=1); - T * wait(int timeOut=-1) - { - T * retval(NULL); - if (empty()) { - _cond.Wait(timeOut); - } - return retval; - } - const T & head() const { return _q[_readPos]; } - void removeHead() { ~_q[_readPos](); _readPos++; } - bool deque(); -private: - std::vector<T> _q; - size_t _readPos; - size_t _writePos; - FastOS_Condition _cond; }; -#endif - } // namespace document diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index e49cf021fe3..29db9e318d8 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -21,6 +21,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -35,14 +36,10 @@ public class DocumentProtocol implements Protocol { private final RoutableRepository routableRepository; private final DocumentTypeManager docMan; - /** - * The name of this protocol. - */ + /** The name of this protocol. */ public static final Utf8String NAME = new Utf8String("document"); - /** - * All message types that are implemented by this protocol. - */ + // All message types that are implemented by this protocol. public static final int DOCUMENT_MESSAGE = 100000; public static final int MESSAGE_GETDOCUMENT = DOCUMENT_MESSAGE + 3; public static final int MESSAGE_PUTDOCUMENT = DOCUMENT_MESSAGE + 4; @@ -62,9 +59,7 @@ public class DocumentProtocol implements Protocol { public static final int MESSAGE_REMOVELOCATION = DOCUMENT_MESSAGE + 24; public static final int MESSAGE_QUERYRESULT = DOCUMENT_MESSAGE + 25; - /** - * All reply types that are implemented by this protocol. - */ + // All reply types that are implemented by this protocol. public static final int DOCUMENT_REPLY = 200000; public static final int REPLY_GETDOCUMENT = DOCUMENT_REPLY + 3; public static final int REPLY_PUTDOCUMENT = DOCUMENT_REPLY + 4; @@ -86,7 +81,7 @@ public class DocumentProtocol implements Protocol { public static final int REPLY_WRONGDISTRIBUTION = DOCUMENT_REPLY + 1000; public static final int REPLY_DOCUMENTIGNORED = DOCUMENT_REPLY + 1001; - /** + /* * Important note on adding new error codes to the Document protocol: * * Changes to this protocol must be reflected in both the Java and C++ versions @@ -95,117 +90,94 @@ public class DocumentProtocol implements Protocol { * longer be guaranteed. */ - /** - * Used by policies to indicate an inappropriate message. - */ + /** Used by policies to indicate an inappropriate message. */ public static final int ERROR_MESSAGE_IGNORED = ErrorCode.APP_FATAL_ERROR + 1; - /** - * Used for error policy when policy creation failed. - */ + + /** Used for error policy when policy creation failed. */ public static final int ERROR_POLICY_FAILURE = ErrorCode.APP_FATAL_ERROR + 2; - /** - * Document in operation cannot be found. (VDS Get and Remove) - */ + + /** Document in operation cannot be found. (VDS Get and Remove) */ public static final int ERROR_DOCUMENT_NOT_FOUND = ErrorCode.APP_FATAL_ERROR + 1001; - /** - * Operation cannot be performed because token already exist. (Create bucket, create visitor) - */ + + /** Operation cannot be performed because token already exist. (Create bucket, create visitor) */ public static final int ERROR_DOCUMENT_EXISTS = ErrorCode.APP_FATAL_ERROR + 1002; - /** - * Node have not implemented support for the given operation. - */ + + /** Node have not implemented support for the given operation. */ public static final int ERROR_NOT_IMPLEMENTED = ErrorCode.APP_FATAL_ERROR + 1004; - /** - * Parameters given in request is illegal. - */ + + /** Parameters given in request is illegal. */ public static final int ERROR_ILLEGAL_PARAMETERS = ErrorCode.APP_FATAL_ERROR + 1005; - /** - * Unknown request received. (New client requesting from old server) - */ + + /** Unknown request received. (New client requesting from old server) */ public static final int ERROR_UNKNOWN_COMMAND = ErrorCode.APP_FATAL_ERROR + 1007; - /** - * Request cannot be decoded. - */ + + /** Request cannot be decoded. */ public static final int ERROR_UNPARSEABLE = ErrorCode.APP_FATAL_ERROR + 1008; - /** - * Not enough free space on disk to perform operation. - */ + + /** Not enough free space on disk to perform operation. */ public static final int ERROR_NO_SPACE = ErrorCode.APP_FATAL_ERROR + 1009; - /** - * Request was not handled correctly. - */ + + /** Request was not handled correctly. */ public static final int ERROR_IGNORED = ErrorCode.APP_FATAL_ERROR + 1010; - /** - * We failed in some way we didn't expect to fail. - */ + + /** We failed in some way we didn't expect to fail. */ public static final int ERROR_INTERNAL_FAILURE = ErrorCode.APP_FATAL_ERROR + 1011; - /** - * Node refuse to perform operation. (Illegally formed message?) - */ + + /** Node refuse to perform operation. (Illegally formed message?) */ public static final int ERROR_REJECTED = ErrorCode.APP_FATAL_ERROR + 1012; - /** - * Test and set condition (selection) failed. - */ + + /** Test and set condition (selection) failed. */ public static final int ERROR_TEST_AND_SET_CONDITION_FAILED = ErrorCode.APP_FATAL_ERROR + 1013; - /** - * Failed to process the given request. (Used by docproc) - */ + /** Failed to process the given request. (Used by docproc) */ public static final int ERROR_PROCESSING_FAILURE = ErrorCode.APP_FATAL_ERROR + 2001; + /** Unique timestamp specified for new operation is already in use. */ public static final int ERROR_TIMESTAMP_EXIST = ErrorCode.APP_FATAL_ERROR + 2002; - /** - * Node not ready to perform operation. (Initializing VDS nodes) - */ + + /** Node not ready to perform operation. (Initializing VDS nodes) */ public static final int ERROR_NODE_NOT_READY = ErrorCode.APP_TRANSIENT_ERROR + 1001; - /** - * Wrong node to talk to in current state. (VDS system state disagreement) - */ + + /** Wrong node to talk to in current state. (VDS system state disagreement) */ public static final int ERROR_WRONG_DISTRIBUTION = ErrorCode.APP_TRANSIENT_ERROR + 1002; - /** - * Operation cut short and aborted. (Destroy visitor, node stopping) - */ + + /** Operation cut short and aborted. (Destroy visitor, node stopping) */ public static final int ERROR_ABORTED = ErrorCode.APP_TRANSIENT_ERROR + 1004; - /** - * Node too busy to process request (Typically full queues) - */ + + /** Node too busy to process request (Typically full queues) */ public static final int ERROR_BUSY = ErrorCode.APP_TRANSIENT_ERROR + 1005; - /** - * Lost connection with the node we requested something from. - */ + + /** Lost connection with the node we requested something from. */ public static final int ERROR_NOT_CONNECTED = ErrorCode.APP_TRANSIENT_ERROR + 1006; - /** - * We failed accessing the disk, which we think is a disk hardware problem. - */ + + /** We failed accessing the disk, which we think is a disk hardware problem. */ public static final int ERROR_DISK_FAILURE = ErrorCode.APP_TRANSIENT_ERROR + 1007; - /** - * We failed during an IO operation, we dont think is a specific disk hardware problem. - */ + + /** We failed during an IO operation, we dont think is a specific disk hardware problem. */ public static final int ERROR_IO_FAILURE = ErrorCode.APP_TRANSIENT_ERROR + 1008; + /** * Bucket given in operation not found due to bucket database * inconsistencies between storage and distributor nodes. */ public static final int ERROR_BUCKET_NOT_FOUND = ErrorCode.APP_TRANSIENT_ERROR + 1009; + /** * Bucket recently removed, such that operation cannot be performed. * Differs from BUCKET_NOT_FOUND in that there is no db inconsistency. */ public static final int ERROR_BUCKET_DELETED = ErrorCode.APP_TRANSIENT_ERROR + 1012; - /** - * Storage node received a timestamp that is stale. Likely clock skew. - */ + + /** Storage node received a timestamp that is stale. Likely clock skew. */ public static final int ERROR_STALE_TIMESTAMP = ErrorCode.APP_TRANSIENT_ERROR + 1013; - /** - * The given node have gotten a critical error and have suspended itself. - */ + /** The given node have gotten a critical error and have suspended itself. */ public static final int ERROR_SUSPENDED = ErrorCode.APP_TRANSIENT_ERROR + 2001; /** - * <p>Define the different priorities allowed for document api messages. Most user traffic should be fit into the + * Defines the different priorities allowed for document api messages. Most user traffic should be fit into the * NORMAL categories. Traffic in the HIGH end will be usually be prioritized over important maintenance operations. - * Traffic in the LOW end will be prioritized after these operations.</p> + * Traffic in the LOW end will be prioritized after these operations. */ public enum Priority { HIGHEST(0), @@ -239,9 +211,9 @@ public class DocumentProtocol implements Protocol { /** * Get a priority enum instance by its value. * - * @param val The value of the priority to return. - * @return The priority enum instance. - * @throws IllegalArgumentException If priority value is unknown. + * @param val the value of the priority to return + * @return the priority enum instance + * @throws IllegalArgumentException if the priority value is unknown */ public static Priority getPriority(int val) { for (Priority pri : Priority.values()) { @@ -255,9 +227,9 @@ public class DocumentProtocol implements Protocol { /** * Get priority enum instance by its name. * - * @param name Name of priority. - * @return Priority enum instance, given that <code>name</code> is valid. - * @throws IllegalArgumentException If priority name is unknown. + * @param name name of priority. + * @return priority enum instance, given that <code>name</code> is valid + * @throws IllegalArgumentException if priority name is unknown */ public static Priority getPriorityByName(String name) { return Priority.valueOf(name); @@ -344,9 +316,9 @@ public class DocumentProtocol implements Protocol { * that is already in use by a message bus instance. Notice that the name you supply for a factory is the * case-sensitive name that will be referenced by routes. * - * @param name The name of the factory to add. - * @param factory The factory to add. - * @return This, to allow chaining. + * @param name the name of the factory to add + * @param factory the factory to add + * @return this, to allow chaining */ public DocumentProtocol putRoutingPolicyFactory(String name, RoutingPolicyFactory factory) { routingPolicyRepository.putFactory(name, factory); @@ -359,10 +331,10 @@ public class DocumentProtocol implements Protocol { * supported version. You can always bypass this by passing a default version specification object to this function, * because that object will match any version. * - * @param type The routable type to assign a factory to. - * @param factory The factory to add. - * @param version The version for which this factory can be used. - * @return This, to allow chaining. + * @param type the routable type to assign a factory to + * @param factory the factory to add + * @param version the version for which this factory can be used + * @return this, to allow chaining */ public DocumentProtocol putRoutableFactory(int type, RoutableFactory factory, VersionSpecification version) { routableRepository.putFactory(version, type, factory); @@ -373,10 +345,10 @@ public class DocumentProtocol implements Protocol { * Convenience method to call {@link #putRoutableFactory(int, RoutableFactory, com.yahoo.component.VersionSpecification)} * for multiple version specifications. * - * @param type The routable type to assign a factory to. - * @param factory The factory to add. - * @param versions The versions for which this factory can be used. - * @return This, to allow chaining. + * @param type the routable type to assign a factory to + * @param factory the factory to add + * @param versions the versions for which this factory can be used + * @return this, to allow chaining */ public DocumentProtocol putRoutableFactory(int type, RoutableFactory factory, List<VersionSpecification> versions) { for (VersionSpecification version : versions) { @@ -454,7 +426,7 @@ public class DocumentProtocol implements Protocol { * This is a convenient entry to the {@link #merge(RoutingContext,Set)} method by way of a routing context object. * The replies of all child contexts are merged and stored in the context. * - * @param ctx The context whose children to merge. + * @param ctx the context whose children to merge */ public static void merge(RoutingContext ctx) { merge(ctx, new HashSet<Integer>(0)); @@ -465,8 +437,8 @@ public class DocumentProtocol implements Protocol { * in any of the replies, it will prepare an EmptyReply() and add all errors to it. If there are no errors, this * method will use the first reply in the list and transfer whatever feed answers might exist in the replies to it. * - * @param ctx The context whose children to merge. - * @param mask The indexes of the children to skip. + * @param ctx the context whose children to merge + * @param mask the indexes of the children to skip */ public static void merge(RoutingContext ctx, Set<Integer> mask) { List<Reply> replies = new LinkedList<>(); @@ -499,8 +471,8 @@ public class DocumentProtocol implements Protocol { * method will use the first reply in the list and transfer whatever feed answers might exist in the replies to it. * * - * @param replies The replies to merge. - * @return The merged Reply. + * @param replies the replies to merge + * @return the merged Reply */ public static Reply merge(List<Reply> replies) { return merge(replies, new HashSet<Integer>(0)).second; @@ -509,9 +481,9 @@ public class DocumentProtocol implements Protocol { /** * Returns true if the given reply has at least one error, and all errors are of the given type. * - * @param reply The reply to check for error. - * @param errCode The error code to check for. - * @return Whether or not the reply has only the given error code. + * @param reply the reply to check for error + * @param errCode the error code to check for + * @return whether or not the reply has only the given error code */ public static boolean hasOnlyErrorsOfType(Reply reply, int errCode) { if (!reply.hasErrors()) { @@ -541,8 +513,7 @@ public class DocumentProtocol implements Protocol { try { return routableRepository.decode(docMan, version, data); } catch (RuntimeException e) { - e.printStackTrace(); - log.warning(e.getMessage()); + log.log(Level.WARNING, "Failed to decode document data", e); return null; } } @@ -550,12 +521,13 @@ public class DocumentProtocol implements Protocol { /** * Returns a list of routable types that support the given version. * - * @param version The version to return types for. - * @return The list of supported types. + * @param version the version to return types for + * @return the list of supported types */ public List<Integer> getRoutableTypes(Version version) { return routableRepository.getRoutableTypes(version); } final public DocumentTypeManager getDocumentTypeManager() { return docMan; } + } diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index 7d7153e8461..dcfc0fa5f6e 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -9,6 +9,7 @@ #include <vespa/documentapi/documentapi.h> #include <vespa/vespalib/util/exceptions.h> #include <sstream> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".documentprotocol"); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp index 8b303d10e0a..82cda6c773f 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp @@ -41,9 +41,7 @@ AsyncInitializationPolicy::AsyncInitializationPolicy( { } -AsyncInitializationPolicy::~AsyncInitializationPolicy() -{ -} +AsyncInitializationPolicy::~AsyncInitializationPolicy() = default; void AsyncInitializationPolicy::initSynchronous() @@ -74,7 +72,7 @@ AsyncInitializationPolicy::select(mbus::RoutingContext& context) } { - vespalib::MonitorGuard lock(_lock); + std::lock_guard lock(_lock); if (_state == State::NOT_STARTED || _state == State::FAILED) { // Only 1 task may be queued to the executor at any point in time. @@ -88,7 +86,7 @@ AsyncInitializationPolicy::select(mbus::RoutingContext& context) } if (_state != State::DONE) { - mbus::Reply::UP reply(new mbus::EmptyReply()); + auto reply = std::make_unique<mbus::EmptyReply>(); reply->addError(currentPolicyInitError()); context.setReply(std::move(reply)); return; @@ -117,7 +115,7 @@ AsyncInitializationPolicy::Task::run() using State = AsyncInitializationPolicy::State; - vespalib::MonitorGuard lock(_owner._lock); + std::lock_guard lock(_owner._lock); _owner._error = error; _owner._state = error.empty() ? State::DONE : State::FAILED; } diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h index f056bb3b329..0e30da8e7c8 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h @@ -5,15 +5,15 @@ #include <vespa/messagebus/error.h> #include <vespa/vespalib/util/executor.h> #include <vespa/documentapi/common.h> -#include <vespa/vespalib/util/sync.h> #include <map> +#include <mutex> namespace documentapi { class AsyncInitializationPolicy : public mbus::IRoutingPolicy { public: AsyncInitializationPolicy(const std::map<string, string>& parameters); - virtual ~AsyncInitializationPolicy(); + ~AsyncInitializationPolicy() override; static std::map<string, string> parse(string parameters); @@ -64,7 +64,7 @@ private: friend class Task; std::unique_ptr<vespalib::Executor> _executor; - vespalib::Monitor _lock; + std::mutex _lock; enum class State { NOT_STARTED, diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp index 378f12b5138..01245843e3b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp @@ -60,7 +60,7 @@ DocumentRouteSelectorPolicy::configure(std::unique_ptr<messagebus::protocol::Doc } config[string(route.name)] = selector; } - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _config.swap(config); _error.swap(error); } @@ -68,7 +68,7 @@ DocumentRouteSelectorPolicy::configure(std::unique_ptr<messagebus::protocol::Doc const string & DocumentRouteSelectorPolicy::getError() const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); return _error; } @@ -84,7 +84,7 @@ DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context) // Invoke private select method for each candidate recipient. { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (!_error.empty()) { context.setError(DocumentProtocol::ERROR_POLICY_FAILURE, _error); return; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h index 6e13f615a35..02e8e4ff282 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h @@ -5,7 +5,6 @@ #include <vespa/document/select/node.h> #include <map> #include <vespa/messagebus/routing/iroutingpolicy.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/documentapi/common.h> #include <vespa/config/config.h> #include <vespa/config/helper/configfetcher.h> @@ -33,7 +32,7 @@ private: typedef std::map<string, SelectorPtr> ConfigMap; const document::DocumentTypeRepo &_repo; - vespalib::Lock _lock; + mutable std::mutex _lock; ConfigMap _config; string _error; config::ConfigFetcher _fetcher; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp index 6c4523fa34f..25112a00b99 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp @@ -118,7 +118,7 @@ ExternPolicy::merge(mbus::RoutingContext &ctx) mbus::Hop ExternPolicy::getRecipient() { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); update(); if (_recipients.empty()) { return mbus::Hop(); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.h index c56f8f214ec..cb0c4428e60 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.h @@ -4,9 +4,8 @@ #include <vespa/messagebus/routing/hop.h> #include <vespa/messagebus/routing/iroutingpolicy.h> #include <vespa/slobrok/imirrorapi.h> -#include <vector> -#include <vespa/vespalib/util/sync.h> #include <vespa/documentapi/common.h> +#include <mutex> class FRT_Supervisor; class FNET_Transport; @@ -21,7 +20,7 @@ namespace documentapi { class ExternPolicy : public mbus::IRoutingPolicy { private: using IMirrorAPI = slobrok::api::IMirrorAPI; - vespalib::Lock _lock; + std::mutex _lock; std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _orb; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp index e82a184d8b2..800aa8c4520 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp @@ -2,6 +2,7 @@ #include "externslobrokpolicy.h" #include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/config/common/configcontext.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/util/time.h> #include <vespa/fnet/frt/frt.h> @@ -61,8 +62,8 @@ string ExternSlobrokPolicy::init() { } else if (_configSources.size() != 0) { slobrok::ConfiguratorFactory config( config::ConfigUri(_slobrokConfigId, - std::make_unique<config::ConfigContext>(config::ServerSpec(_configSources)))); - _mirror.reset(new MirrorAPI(*_orb, config)); + std::make_shared<config::ConfigContext>(config::ServerSpec(_configSources)))); + _mirror = std::make_unique<MirrorAPI>(*_orb, config); } if (_mirror.get()) { @@ -74,7 +75,7 @@ string ExternSlobrokPolicy::init() { IMirrorAPI::SpecList ExternSlobrokPolicy::lookup(mbus::RoutingContext& context, const string& pattern) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); const IMirrorAPI& mirror(_mirror.get()? *_mirror : context.getMirror()); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.h index d2966f852d5..865709daf7f 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.h @@ -23,7 +23,7 @@ class ExternSlobrokPolicy : public AsyncInitializationPolicy protected: bool _firstTry; config::ServerSpec::HostSpecList _configSources; - vespalib::Lock _lock; + std::mutex _lock; std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _orb; @@ -33,7 +33,7 @@ protected: public: ExternSlobrokPolicy(const std::map<string, string>& params); - ~ExternSlobrokPolicy(); + ~ExternSlobrokPolicy() override; /** * @return a pointer to the slobrok mirror owned by this policy, if any. diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp index 7db941a12f2..9b23d593e82 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp @@ -49,7 +49,7 @@ LocalServicePolicy::getCacheKey(const mbus::RoutingContext &ctx) const mbus::Hop LocalServicePolicy::getRecipient(mbus::RoutingContext &ctx) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); CacheEntry &entry = update(ctx); if (entry._recipients.empty()) { mbus::Hop hop = ctx.getRoute().getHop(0); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.h index 4dd22a77069..c188c9f5c4b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.h @@ -4,9 +4,9 @@ #include <vespa/documentapi/common.h> #include <vespa/messagebus/routing/hop.h> #include <vespa/messagebus/routing/iroutingpolicy.h> -#include <vespa/vespalib/util/sync.h> #include <vector> #include <map> +#include <mutex> namespace documentapi { @@ -26,7 +26,7 @@ private: CacheEntry(); }; - vespalib::Lock _lock; + std::mutex _lock; string _address; std::map<string, CacheEntry> _cache; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp index 6f778a3e4d0..a58f3439df2 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp @@ -56,7 +56,7 @@ RoundRobinPolicy::getCacheKey(const mbus::RoutingContext &ctx) const mbus::Hop RoundRobinPolicy::getRecipient(mbus::RoutingContext &ctx) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); CacheEntry &entry = update(ctx); if (entry._recipients.empty()) { return mbus::Hop(); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.h index 0bfd2f04ea7..98e61fd1a1c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.h @@ -4,8 +4,8 @@ #include <vespa/documentapi/common.h> #include <vespa/messagebus/routing/hop.h> #include <vespa/messagebus/routing/iroutingpolicy.h> -#include <vespa/vespalib/util/sync.h> #include <map> +#include <mutex> namespace documentapi { @@ -25,7 +25,7 @@ private: CacheEntry(); }; - vespalib::Lock _lock; + std::mutex _lock; std::map<string, CacheEntry> _cache; /** diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp index 166c3be5e66..e49d412fee1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/config-stor-distribution.h> #include <vespa/config/subscription/configuri.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".storagepolicy"); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp index 7e324468ae5..375cdc23ba2 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp @@ -64,7 +64,7 @@ SubsetServicePolicy::getRecipient(mbus::RoutingContext &ctx) { mbus::Hop hop; if (_subsetSize > 0) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); CacheEntry &entry = update(ctx); if (!entry._recipients.empty()) { if (++entry._offset >= entry._recipients.size()) { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.h index bf44afa318f..a6c2e0d6cae 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.h @@ -4,9 +4,8 @@ #include <vespa/documentapi/common.h> #include <vespa/messagebus/routing/hop.h> #include <vespa/messagebus/routing/iroutingpolicy.h> -#include <vespa/vespalib/util/sync.h> -#include <vector> #include <map> +#include <mutex> namespace documentapi { @@ -26,8 +25,8 @@ private: CacheEntry(); }; - vespalib::Lock _lock; - uint32_t _subsetSize; + std::mutex _lock; + uint32_t _subsetSize; std::map<string, CacheEntry> _cache; /** diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp index c7f3401d3e1..6ed33cda060 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp @@ -109,7 +109,7 @@ void RoutableRepository::putFactory(const vespalib::VersionSpecification &version, uint32_t type, IRoutableFactory::SP factory) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (_factoryTypes[type].putFactory(version, factory)) { _cache.clear(); } @@ -118,7 +118,7 @@ RoutableRepository::putFactory(const vespalib::VersionSpecification &version, IRoutableFactory::SP RoutableRepository::getFactory(const vespalib::Version &version, uint32_t type) const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); CacheKey cacheKey(version, type); FactoryCache::const_iterator cit = _cache.find(cacheKey); if (cit != _cache.end()) { @@ -139,7 +139,7 @@ RoutableRepository::getFactory(const vespalib::Version &version, uint32_t type) uint32_t RoutableRepository::getRoutableTypes(const vespalib::Version &version, std::vector<uint32_t> &out) const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); for (const auto & type : _factoryTypes) { if (type.second.getFactory(version)) { out.push_back(type.first); diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.h b/documentapi/src/vespa/documentapi/messagebus/routablerepository.h index f4bddad45b4..7e73f3929b8 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablerepository.h +++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.h @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <map> +#include "iroutablefactory.h" #include <vespa/messagebus/blobref.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/component/versionspecification.h> -#include "iroutablefactory.h" +#include <mutex> +#include <map> namespace documentapi { @@ -37,7 +37,7 @@ private: typedef std::map<CacheKey, IRoutableFactory::SP> FactoryCache; typedef std::map<uint32_t, VersionMap> TypeMap; - vespalib::Lock _lock; + mutable std::mutex _lock; TypeMap _factoryTypes; mutable FactoryCache _cache; const LoadTypeSet& _loadTypes; @@ -48,7 +48,7 @@ public: /** * Constructs a new routable repository. */ - RoutableRepository(const LoadTypeSet& loadTypes); + explicit RoutableRepository(const LoadTypeSet& loadTypes); /** * Decodes a {@link Routable} from the given byte array. This uses the content of the byte array to diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.cpp index 3c73cda786d..c21d8e97880 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.cpp @@ -17,14 +17,14 @@ RoutingPolicyRepository::RoutingPolicyRepository() : void RoutingPolicyRepository::putFactory(const string &name, IRoutingPolicyFactory::SP factory) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _factories[name] = factory; } IRoutingPolicyFactory::SP RoutingPolicyRepository::getFactory(const string &name) const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); FactoryMap::const_iterator it = _factories.find(name); if (it != _factories.end()) { return it->second; diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.h b/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.h index ca6057ef8b1..beeb176c88b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.h +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.h @@ -2,8 +2,8 @@ #pragma once #include "iroutingpolicyfactory.h" -#include <vespa/vespalib/util/sync.h> #include <map> +#include <mutex> namespace documentapi { @@ -11,8 +11,8 @@ class RoutingPolicyRepository { private: typedef std::map<string, IRoutingPolicyFactory::SP> FactoryMap; - vespalib::Lock _lock; - FactoryMap _factories; + mutable std::mutex _lock; + FactoryMap _factories; public: RoutingPolicyRepository(const RoutingPolicyRepository &) = delete; diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-MapVisitorMessage.dat Binary files differindex 1a8a837ea16..c9201fe780d 100644 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-MapVisitorMessage.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-MapVisitorMessage.dat diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 18c3676c366..230c90009f5 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_define_module( src/tests/eval/aggr src/tests/eval/compile_cache src/tests/eval/compiled_function + src/tests/eval/fast_sparse_map src/tests/eval/function src/tests/eval/function_speed src/tests/eval/gbdt @@ -24,7 +25,6 @@ vespa_define_module( src/tests/eval/node_tools src/tests/eval/node_types src/tests/eval/param_usage - src/tests/eval/simple_sparse_map src/tests/eval/simple_tensor src/tests/eval/simple_value src/tests/eval/tensor_function @@ -65,6 +65,9 @@ vespa_define_module( src/tests/tensor/instruction_benchmark src/tests/tensor/onnx_wrapper src/tests/tensor/packed_mappings + src/tests/tensor/partial_add + src/tests/tensor/partial_modify + src/tests/tensor/partial_remove src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 59fe2960dbb..6099e543922 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -278,8 +278,8 @@ void compare_test(const Inspector &expect_in, const Inspector &actual_in) { void compare(Input &expect, Input &actual) { TestList expect_tests; TestList actual_tests; - for_each_test(expect, std::bind(&TestList::add_test, &expect_tests, _1), [](Slime &){}); - for_each_test(actual, std::bind(&TestList::add_test, &actual_tests, _1), [](Slime &){}); + for_each_test(expect, std::bind(&TestList::add_test, &expect_tests, _1), [](Slime &) noexcept {}); + for_each_test(actual, std::bind(&TestList::add_test, &actual_tests, _1), [](Slime &) noexcept {}); ASSERT_TRUE(!expect_tests.list.empty()); ASSERT_TRUE(!actual_tests.list.empty()); ASSERT_EQUAL(expect_tests.list.size(), actual_tests.list.size()); diff --git a/eval/src/tests/ann/hnsw-like.h b/eval/src/tests/ann/hnsw-like.h index 841957c1ccb..ac4a346993d 100644 --- a/eval/src/tests/ann/hnsw-like.h +++ b/eval/src/tests/ann/hnsw-like.h @@ -86,7 +86,7 @@ struct VisitedSetPool struct HnswHit { double dist; uint32_t docid; - HnswHit(uint32_t di, SqDist sq) : dist(sq.distance), docid(di) {} + HnswHit(uint32_t di, SqDist sq) noexcept : dist(sq.distance), docid(di) {} }; struct GreaterDist { diff --git a/eval/src/tests/ann/nns.h b/eval/src/tests/ann/nns.h index 7a20a132248..6351733fb59 100644 --- a/eval/src/tests/ann/nns.h +++ b/eval/src/tests/ann/nns.h @@ -8,13 +8,13 @@ struct SqDist { double distance; - explicit SqDist(double d) : distance(d) {} + explicit SqDist(double d) noexcept : distance(d) {} }; struct NnsHit { uint32_t docid; SqDist sq; - NnsHit(uint32_t di, SqDist sqD) + NnsHit(uint32_t di, SqDist sqD) noexcept : docid(di), sq(sqD) {} }; struct NnsHitComparatorLessDistance { diff --git a/eval/src/tests/eval/fast_sparse_map/CMakeLists.txt b/eval/src/tests/eval/fast_sparse_map/CMakeLists.txt new file mode 100644 index 00000000000..fd9c1d37d43 --- /dev/null +++ b/eval/src/tests/eval/fast_sparse_map/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_fast_sparse_map_test_app TEST + SOURCES + fast_sparse_map_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_fast_sparse_map_test_app COMMAND eval_fast_sparse_map_test_app) diff --git a/eval/src/tests/eval/simple_sparse_map/simple_sparse_map_test.cpp b/eval/src/tests/eval/fast_sparse_map/fast_sparse_map_test.cpp index 8062881ced9..3d98ec67aa8 100644 --- a/eval/src/tests/eval/simple_sparse_map/simple_sparse_map_test.cpp +++ b/eval/src/tests/eval/fast_sparse_map/fast_sparse_map_test.cpp @@ -1,6 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/eval/eval/simple_sparse_map.h> +#include <vespa/eval/eval/fast_sparse_map.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/gtest/gtest.h> @@ -24,19 +24,30 @@ public: } } ~StringList(); - const std::vector<vespalib::string> direct_str() const { return _str_list; } + ConstArrayRef<vespalib::string> direct_str() const { return _str_list; } ConstArrayRef<vespalib::stringref> direct_ref() const { return _ref_list; } ConstArrayRef<const vespalib::stringref *> indirect_ref() const { return _ref_ptr_list; } + bool is_eq(ConstArrayRef<FastSparseMap::HashedLabel> addr) const { + if (addr.size() != _str_list.size()) { + return false; + } + for (size_t i = 0; i < addr.size(); ++i) { + if (addr[i].label != _str_list[i]) { + return false; + } + } + return true; + } }; StringList::~StringList() = default; using SL = StringList; -TEST(SimpleSparseMapTest, simple_sparse_map_basic_usage_works) { +TEST(FastSparseMapTest, fast_sparse_map_basic_usage_works) { SL a1({"a","a","a"}); SL a2({"a","a","b"}); SL a3({"a","b","a"}); SL a4({"b","a","a"}); - SimpleSparseMap map(3, 128); + FastSparseMap map(3, 128); EXPECT_EQ(map.size(), 0); map.add_mapping(a1.direct_str()); map.add_mapping(a2.direct_ref()); @@ -54,18 +65,34 @@ TEST(SimpleSparseMapTest, simple_sparse_map_basic_usage_works) { EXPECT_EQ(map.lookup(a4.direct_str()), map.npos()); EXPECT_EQ(map.lookup(a4.direct_ref()), map.npos()); EXPECT_EQ(map.lookup(a4.indirect_ref()), map.npos()); - EXPECT_EQ(SimpleSparseMap::npos(), map.npos()); - SL expect_labels({"a","a","a", - "a","a","b", - "a","b","a"}); - EXPECT_EQ(map.labels(), expect_labels.direct_str()); + EXPECT_EQ(FastSparseMap::npos(), map.npos()); + EXPECT_EQ(map.labels().size(), 9); + std::set<uint64_t> seen_hashes; + std::map<uint32_t, uint32_t> addr_map; + auto my_fun = [&](uint32_t subspace, uint64_t hash) { + uint32_t addr_tag = subspace * 3; + addr_map[addr_tag] = subspace; + seen_hashes.insert(hash); + }; + map.each_map_entry(my_fun); + EXPECT_EQ(seen_hashes.size(), 3); + EXPECT_EQ(addr_map.size(), 3); + EXPECT_NE(addr_map.find(0), addr_map.end()); + EXPECT_EQ(addr_map[0], 0); + EXPECT_EQ(addr_map[3], 1); + EXPECT_EQ(addr_map[6], 2); + EXPECT_EQ(addr_map.size(), 3); + EXPECT_TRUE(a1.is_eq(map.make_addr(0))); + EXPECT_FALSE(a2.is_eq(map.make_addr(0))); + EXPECT_TRUE(a2.is_eq(map.make_addr(1))); + EXPECT_TRUE(a3.is_eq(map.make_addr(2))); } -TEST(SimpleSparseMapTest, simple_sparse_map_works_with_no_labels) { +TEST(FastSparseMapTest, fast_sparse_map_works_with_no_labels) { SL empty({}); - SimpleSparseMap map1(0, 1); - SimpleSparseMap map2(0, 1); - SimpleSparseMap map3(0, 1); + FastSparseMap map1(0, 1); + FastSparseMap map2(0, 1); + FastSparseMap map3(0, 1); EXPECT_EQ(map1.size(), 0); EXPECT_EQ(map2.size(), 0); EXPECT_EQ(map3.size(), 0); @@ -89,8 +116,8 @@ TEST(SimpleSparseMapTest, simple_sparse_map_works_with_no_labels) { EXPECT_EQ(map3.labels().size(), 0); } -TEST(SimpleSparseMapTest, size_of_internal_types) { - fprintf(stderr, "simple sparse map hash node size: %zu\n", sizeof(hash_node<SimpleSparseMap::MapType::value_type>)); +TEST(FastSparseMapTest, size_of_internal_types) { + fprintf(stderr, "fast sparse map hash node size: %zu\n", sizeof(hash_node<FastSparseMap::MapType::value_type>)); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/simple_sparse_map/CMakeLists.txt b/eval/src/tests/eval/simple_sparse_map/CMakeLists.txt deleted file mode 100644 index e3d2885e426..00000000000 --- a/eval/src/tests/eval/simple_sparse_map/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_simple_sparse_map_test_app TEST - SOURCES - simple_sparse_map_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_simple_sparse_map_test_app COMMAND eval_simple_sparse_map_test_app) diff --git a/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp b/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp index a2510fdd2fa..dc69bce30f6 100644 --- a/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp +++ b/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp @@ -20,29 +20,40 @@ using vespalib::make_string_short::fmt; std::vector<Layout> concat_layouts = { {}, {}, - {}, {x(5)}, - {x(5)}, {}, - {x(2)}, {x(3)}, - {x(2)}, {y(3)}, - {y(2)}, {z(3)}, - {x(5)}, {x(2),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(2)}, {x(3),y(5),z(2)}, - {x(2),y(5),z(2)}, {x(3),y(5),z(2)}, - {x(3),y(5)}, {y(5),z(7)}, - float_cells({x(3),y(5)}), {y(5),z(7)}, - {x(3),y(5)}, float_cells({y(5),z(7)}), - float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), - {y({"a","b","c"})}, {y({"a","b","c"})}, - {y({"a","b","c"})}, {y({"a","b"})}, - {y({"a","b","c"})}, {y({"b","c","d"})}, - float_cells({y({"a","b","c"})}), {y({"b","c","d"})}, - {y({"a","b","c"})}, float_cells({y({"b","c","d"})}), - float_cells({y({"a","b","c"})}), float_cells({z({"foo","bar","baz"})}), - {y({"a","b","c"})}, {y({"a","b","c"}),z({"foo","bar","baz"})}, - {y({"a","b"}),z({"foo","bar","baz"})}, {y({"a","b","c"}),z({"foo","bar"})}, - {x(2),y({"a","b","c"})}, {x(3),y({"b","c","d"})}, - {x(2),y({"a","b"})}, {x(3),z({"c","d"})} + {}, {y(5)}, + float_cells({y(5)}), {}, + {}, float_cells({y(5)}), + {y(5)}, {}, + {y(2)}, {y(3)}, + {y(2)}, {x(3)}, + {x(2)}, {z(3)}, + {x(2),y(3)}, {x(2),y(3)}, + {x(2),y(3)}, {x(2),y(4)}, + {y(3),z(5)}, {y(3),z(5)}, + {y(3),z(5)}, {y(4),z(5)}, + {x(2),y(3),z(5)}, {x(2),y(3),z(5)}, + {x(2),y(3),z(5)}, {x(2),y(4),z(5)}, + {x(2),y(3),z({"a","b"})}, {x(2),y(3),z({"b","c"})}, + {x(2),y(3),z({"a","b"})}, {x(2),y(4),z({"b","c"})}, + {y(5)}, {y(2),x(5)}, + {x(3)}, {y(2),z(3)}, + {y(2)}, {y(3),x(5),z(2)}, + {y(2),x(5),z(2)}, {y(3),x(5),z(2)}, + {y(3),x(5)}, {x(5),z(7)}, + float_cells({y(3),x(5)}), {x(5),z(7)}, + float_cells({y(3),x(5)}), {}, + {y(3),x(5)}, float_cells({x(5),z(7)}), + float_cells({y(3),x(5)}), float_cells({x(5),z(7)}), + {x({"a","b","c"})}, {x({"a","b","c"})}, + {x({"a","b","c"})}, {x({"a","b"})}, + {x({"a","b","c"})}, {x({"b","c","d"})}, + float_cells({x({"a","b","c"})}), {x({"b","c","d"})}, + {x({"a","b","c"})}, float_cells({x({"b","c","d"})}), + float_cells({x({"a","b","c"})}), float_cells({z({"foo","bar","baz"})}), + {x({"a","b","c"})}, {x({"a","b","c"}),z({"foo","bar","baz"})}, + {x({"a","b"}),z({"foo","bar","baz"})}, {x({"a","b","c"}),z({"foo","bar"})}, + {y(2),x({"a","b","c"})}, {y(3),x({"b","c","d"})}, + {y(2),x({"a","b"})}, {y(3),z({"c","d"})} }; TensorSpec perform_simpletensor_concat(const TensorSpec &a, const TensorSpec &b, const std::string &dimension) { @@ -101,8 +112,8 @@ TensorSpec reference_concat(const TensorSpec &a, const TensorSpec &b, const std: TensorSpec::Address addr_a; TensorSpec::Address addr_b; if (concat_addresses(cell_a.first, cell_b.first, concat_dim, b_offset, addr_a, addr_b)) { - result.set(addr_a, cell_a.second); - result.set(addr_b, cell_b.second); + result.add(addr_a, cell_a.second); + result.add(addr_b, cell_b.second); } } } @@ -125,8 +136,8 @@ TEST(GenericConcatTest, generic_reference_concat_works) { const TensorSpec lhs = spec(concat_layouts[i], N()); const TensorSpec rhs = spec(concat_layouts[i + 1], Div16(N())); SCOPED_TRACE(fmt("\n===\nin LHS: %s\nin RHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto actual = reference_concat(lhs, rhs, "x"); - auto expect = perform_simpletensor_concat(lhs, rhs, "x"); + auto actual = reference_concat(lhs, rhs, "y"); + auto expect = perform_simpletensor_concat(lhs, rhs, "y"); EXPECT_EQ(actual, expect); } } @@ -137,10 +148,33 @@ TEST(GenericConcatTest, generic_concat_works_for_simple_values) { const TensorSpec lhs = spec(concat_layouts[i], N()); const TensorSpec rhs = spec(concat_layouts[i + 1], Div16(N())); SCOPED_TRACE(fmt("\n===\nin LHS: %s\nin RHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto actual = perform_generic_concat(lhs, rhs, "x"); - auto expect = reference_concat(lhs, rhs, "x"); + auto actual = perform_generic_concat(lhs, rhs, "y"); + auto expect = reference_concat(lhs, rhs, "y"); EXPECT_EQ(actual, expect); } } +TEST(GenericConcatTest, dense_concat_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a[2],b[3],c[5],d{},f[2],g[3])"); + auto rhs = ValueType::from_spec("tensor(a[2],b[3],c[7],e{},h[3],i[4])"); + auto res_type = ValueType::concat(lhs, rhs, "c"); + auto plan = DenseConcatPlan(lhs, rhs, "c", res_type); + EXPECT_EQ(plan.right_offset, 5*2*3*3*4); + EXPECT_EQ(plan.output_size, 2*3*12*2*3*3*4); + EXPECT_EQ(plan.left.input_size, 2*3*5*2*3); + std::vector<size_t> expect_left_loop = { 6, 5, 6, 12 }; + std::vector<size_t> expect_left_in_s = { 30, 6, 1, 0 }; + std::vector<size_t> expect_left_out_s = { 864, 72, 12, 1 }; + EXPECT_EQ(plan.left.in_loop_cnt, expect_left_loop); + EXPECT_EQ(plan.left.in_stride, expect_left_in_s); + EXPECT_EQ(plan.left.out_stride, expect_left_out_s); + EXPECT_EQ(plan.right.input_size, 2*3*7*3*4); + std::vector<size_t> expect_right_loop = { 6, 7, 6, 12 }; + std::vector<size_t> expect_right_in_s = { 84, 12, 0, 1 }; + std::vector<size_t> expect_right_out_s = { 864, 72, 12, 1 }; + EXPECT_EQ(plan.right.in_loop_cnt, expect_right_loop); + EXPECT_EQ(plan.right.in_stride, expect_right_in_s); + EXPECT_EQ(plan.right.out_stride, expect_right_out_s); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/generic_join/generic_join_test.cpp b/eval/src/tests/instruction/generic_join/generic_join_test.cpp index 4821bf092da..25715f07bea 100644 --- a/eval/src/tests/instruction/generic_join/generic_join_test.cpp +++ b/eval/src/tests/instruction/generic_join/generic_join_test.cpp @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_join.h> #include <vespa/eval/eval/interpreted_function.h> @@ -79,6 +80,16 @@ TensorSpec perform_generic_join(const TensorSpec &a, const TensorSpec &b, join_f return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs,*rhs}))); } +TensorSpec perform_generic_join_fast(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + Stash stash; + const auto &factory = FastValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + auto my_op = GenericJoin::make_instruction(lhs->type(), rhs->type(), function, factory, stash); + InterpretedFunction::EvalSingle single(my_op); + return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs,*rhs}))); +} + TEST(GenericJoinTest, dense_join_plan_can_be_created) { auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); auto rhs = ValueType::from_spec("tensor(a{},b[6],c[5],d[4],h{})"); @@ -121,7 +132,7 @@ TEST(GenericJoinTest, dense_join_plan_can_be_executed) { EXPECT_EQ(c, expect); } -TEST(GenericJoinTest, generic_join_works_for_simple_values) { +TEST(GenericJoinTest, generic_join_works_for_simple_and_fast_values) { ASSERT_TRUE((join_layouts.size() % 2) == 0); for (size_t i = 0; i < join_layouts.size(); i += 2) { TensorSpec lhs = spec(join_layouts[i], Div16(N())); @@ -130,7 +141,9 @@ TEST(GenericJoinTest, generic_join_works_for_simple_values) { SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); auto expect = reference_join(lhs, rhs, fun); auto actual = perform_generic_join(lhs, rhs, fun); + auto fast = perform_generic_join_fast(lhs, rhs, fun); EXPECT_EQ(actual, expect); + EXPECT_EQ(fast, expect); } } } diff --git a/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp b/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp index 732fc9c3e69..9ebcb0ec77c 100644 --- a/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp +++ b/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp @@ -16,7 +16,7 @@ using namespace vespalib; const TensorEngine &engine = DefaultTensorEngine::ref(); TypedCells getCellsRef(const eval::Value &value) { - return static_cast<const DenseTensorView &>(value).cellsRef(); + return value.cells(); } struct ChildMock : Leaf { diff --git a/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp b/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp index b0927fd7e90..34e5363d313 100644 --- a/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp +++ b/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp @@ -59,12 +59,12 @@ EvalFixture::ParamRepo make_params() { .add_vector("x", 5) .add_dense({{"c", 5}, {"d", 1}}) .add_dense({{"b", 1}, {"c", 5}}) - .add_matrix("x", 3, "y", 5, [](size_t idx){ return double((idx * 2) + 3); }) - .add_matrix("x", 3, "y", 5, [](size_t idx){ return double((idx * 3) + 2); }) - .add_vector("y", 5, [](size_t idx){ return double((idx * 2) + 3); }) - .add_vector("y", 5, [](size_t idx){ return double((idx * 3) + 2); }) - .add_matrix("y", 5, "z", 3, [](size_t idx){ return double((idx * 2) + 3); }) - .add_matrix("y", 5, "z", 3, [](size_t idx){ return double((idx * 3) + 2); }); + .add_matrix("x", 3, "y", 5, [](size_t idx) noexcept { return double((idx * 2) + 3); }) + .add_matrix("x", 3, "y", 5, [](size_t idx) noexcept { return double((idx * 3) + 2); }) + .add_vector("y", 5, [](size_t idx) noexcept { return double((idx * 2) + 3); }) + .add_vector("y", 5, [](size_t idx) noexcept { return double((idx * 3) + 2); }) + .add_matrix("y", 5, "z", 3, [](size_t idx) noexcept { return double((idx * 2) + 3); }) + .add_matrix("y", 5, "z", 3, [](size_t idx) noexcept { return double((idx * 3) + 2); }); } EvalFixture::ParamRepo param_repo = make_params(); diff --git a/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp b/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp index 75e9c7868ce..4c96145862c 100644 --- a/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp +++ b/eval/src/tests/tensor/direct_dense_tensor_builder/direct_dense_tensor_builder_test.cpp @@ -29,9 +29,8 @@ void assertTensor(const vespalib::string &type_spec, const std::vector<double> &expCells, const Tensor &tensor) { - const DenseTensorView &realTensor = dynamic_cast<const DenseTensorView &>(tensor); - EXPECT_EQUAL(ValueType::from_spec(type_spec), realTensor.type()); - EXPECT_EQUAL(expCells, dispatch_1<CallMakeVector>(realTensor.cellsRef())); + EXPECT_EQUAL(ValueType::from_spec(type_spec), tensor.type()); + EXPECT_EQUAL(expCells, dispatch_1<CallMakeVector>(tensor.cells())); } void assertTensorSpec(const TensorSpec &expSpec, const Tensor &tensor) { diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 2c5b262ee0d..953415f9f71 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -21,7 +21,9 @@ // verifying that all implementations produce the same result. #include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/instruction/generic_concat.h> #include <vespa/eval/instruction/generic_join.h> #include <vespa/eval/instruction/generic_reduce.h> #include <vespa/eval/instruction/generic_rename.h> @@ -66,6 +68,7 @@ struct Impl { virtual Instruction create_reduce(const ValueType &lhs, Aggr aggr, const std::vector<vespalib::string> &dims, Stash &stash) const = 0; virtual Instruction create_rename(const ValueType &lhs, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) const = 0; virtual Instruction create_merge(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const = 0; + virtual Instruction create_concat(const ValueType &lhs, const ValueType &rhs, const std::string &dimension, Stash &stash) const = 0; virtual const TensorEngine &engine() const { return SimpleTensorEngine::ref(); } // engine used by EvalSingle virtual ~Impl() {} }; @@ -88,6 +91,9 @@ struct ValueImpl : Impl { Instruction create_merge(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { return GenericMerge::make_instruction(lhs, rhs, function, my_factory, stash); } + Instruction create_concat(const ValueType &lhs, const ValueType &rhs, const std::string &dimension, Stash &stash) const override { + return GenericConcat::make_instruction(lhs, rhs, dimension, my_factory, stash); + } }; struct EngineImpl : Impl { @@ -122,16 +128,24 @@ struct EngineImpl : Impl { const auto &merge_node = tensor_function::merge(lhs_node, rhs_node, function, stash); return merge_node.compile_self(my_engine, stash); } + Instruction create_concat(const ValueType &lhs, const ValueType &rhs, const std::string &dimension, Stash &stash) const override { + // create a complete tensor function, but only compile the relevant instruction + const auto &lhs_node = tensor_function::inject(lhs, 0, stash); + const auto &rhs_node = tensor_function::inject(rhs, 1, stash); + const auto &concat_node = tensor_function::concat(lhs_node, rhs_node, dimension, stash); + return concat_node.compile_self(my_engine, stash); + } const TensorEngine &engine() const override { return my_engine; } }; //----------------------------------------------------------------------------- -EngineImpl simple_tensor_engine_impl(4, " SimpleTensorEngine", " SimpleT", SimpleTensorEngine::ref()); +EngineImpl simple_tensor_engine_impl(5, " SimpleTensorEngine", " SimpleT", SimpleTensorEngine::ref()); EngineImpl default_tensor_engine_impl(1, "DefaultTensorEngine", "OLD PROD", DefaultTensorEngine::ref()); -ValueImpl simple_value_impl(3, " SimpleValue", " SimpleV", SimpleValueBuilderFactory::get()); -ValueImpl packed_mixed_tensor_impl(2, " PackedMixedTensor", " Packed", PackedMixedTensorBuilderFactory::get()); -ValueImpl default_tensor_value_impl(0, " DefaultValue", "NEW PROD", DefaultValueBuilderFactory::get()); +ValueImpl simple_value_impl(2, " SimpleValue", " SimpleV", SimpleValueBuilderFactory::get()); +ValueImpl fast_value_impl(0, " FastValue", "NEW PROD", FastValueBuilderFactory::get()); +ValueImpl packed_mixed_tensor_impl(4, " PackedMixedTensor", " Packed", PackedMixedTensorBuilderFactory::get()); +ValueImpl default_tensor_value_impl(3, " DefaultValue", "DefaultV", DefaultValueBuilderFactory::get()); vespalib::string short_header("--------"); constexpr double budget = 5.0; @@ -142,6 +156,7 @@ constexpr double good_limit = 1.10; // GOOD: new prod has performance higher tha std::vector<CREF<Impl>> impl_list = {simple_tensor_engine_impl, default_tensor_engine_impl, simple_value_impl, + fast_value_impl, packed_mixed_tensor_impl, default_tensor_value_impl}; @@ -344,6 +359,27 @@ void benchmark_merge(const vespalib::string &desc, const TensorSpec &lhs, //----------------------------------------------------------------------------- +void benchmark_concat(const vespalib::string &desc, const TensorSpec &lhs, + const TensorSpec &rhs, const std::string &dimension) +{ + Stash stash; + ValueType lhs_type = ValueType::from_spec(lhs.type()); + ValueType rhs_type = ValueType::from_spec(rhs.type()); + ValueType res_type = ValueType::concat(lhs_type, rhs_type, dimension); + ASSERT_FALSE(lhs_type.is_error()); + ASSERT_FALSE(rhs_type.is_error()); + ASSERT_FALSE(res_type.is_error()); + std::vector<EvalOp::UP> list; + for (const Impl &impl: impl_list) { + auto op = impl.create_concat(lhs_type, rhs_type, dimension, stash); + std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); + list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + } + benchmark(desc, list); +} + +//----------------------------------------------------------------------------- + struct D { vespalib::string name; bool mapped; @@ -406,6 +442,44 @@ TEST(MakeInputTest, print_some_test_input) { //----------------------------------------------------------------------------- +TEST(DenseConcat, small_vectors) { + auto lhs = make_vector(D::idx("x", 10), 1.0); + auto rhs = make_vector(D::idx("x", 10), 2.0); + benchmark_concat("small dense vector append concat", lhs, rhs, "x"); +} + +TEST(DenseConcat, cross_vectors) { + auto lhs = make_vector(D::idx("x", 10), 1.0); + auto rhs = make_vector(D::idx("x", 10), 2.0); + benchmark_concat("small dense vector cross concat", lhs, rhs, "y"); +} + +TEST(DenseConcat, cube_and_vector) { + auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); + auto rhs = make_vector(D::idx("a", 16), 42.0); + benchmark_concat("cube vs vector concat", lhs, rhs, "a"); +} + +TEST(SparseConcat, small_vectors) { + auto lhs = make_vector(D::map("x", 10, 1), 1.0); + auto rhs = make_vector(D::map("x", 10, 2), 2.0); + benchmark_concat("small sparse concat", lhs, rhs, "y"); +} + +TEST(MixedConcat, large_mixed_a) { + auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 1), 1.0); + auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 2), 2.0); + benchmark_concat("mixed append concat a", lhs, rhs, "a"); +} + +TEST(MixedConcat, large_mixed_b) { + auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 1), 1.0); + auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 2), 2.0); + benchmark_concat("mixed append concat b", lhs, rhs, "b"); +} + +//----------------------------------------------------------------------------- + TEST(NumberJoin, plain_op2) { auto lhs = make_spec(2.0); auto rhs = make_spec(3.0); diff --git a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp index d1d8bc796ba..fce7ccc6411 100644 --- a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp +++ b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp @@ -162,7 +162,7 @@ TEST(OnnxTest, simple_onnx_model_can_be_evaluated) ctx.bind_param(1, attribute); ctx.bind_param(2, bias); ctx.eval(); - auto cells = static_cast<const DenseTensorView&>(output).cellsRef(); + auto cells = output.cells(); EXPECT_EQ(cells.type, ValueType::CellType::FLOAT); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); @@ -208,7 +208,7 @@ TEST(OnnxTest, dynamic_onnx_model_can_be_evaluated) ctx.bind_param(1, attribute); ctx.bind_param(2, bias); ctx.eval(); - auto cells = static_cast<const DenseTensorView&>(output).cellsRef(); + auto cells = output.cells(); EXPECT_EQ(cells.type, ValueType::CellType::FLOAT); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); @@ -254,7 +254,7 @@ TEST(OnnxTest, int_types_onnx_model_can_be_evaluated) ctx.bind_param(1, attribute); ctx.bind_param(2, bias); ctx.eval(); - auto cells = static_cast<const DenseTensorView&>(output).cellsRef(); + auto cells = output.cells(); EXPECT_EQ(cells.type, ValueType::CellType::DOUBLE); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); diff --git a/eval/src/tests/tensor/partial_add/CMakeLists.txt b/eval/src/tests/tensor/partial_add/CMakeLists.txt new file mode 100644 index 00000000000..f0d07a8e9cf --- /dev/null +++ b/eval/src/tests/tensor/partial_add/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_partial_add_test_app TEST + SOURCES + partial_add_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_partial_add_test_app COMMAND eval_partial_add_test_app) diff --git a/eval/src/tests/tensor/partial_add/partial_add_test.cpp b/eval/src/tests/tensor/partial_add/partial_add_test.cpp new file mode 100644 index 00000000000..4546ae42a5e --- /dev/null +++ b/eval/src/tests/tensor/partial_add/partial_add_test.cpp @@ -0,0 +1,119 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/tensor/cell_values.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/partial_update.h> +#include <vespa/eval/tensor/sparse/sparse_tensor.h> +#include <vespa/eval/tensor/tensor.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <optional> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector<Layout> add_layouts = { + {x({"a"})}, {x({"b"})}, + {x({"a","b"})}, {x({"a","c"})}, + float_cells({x({"a","b"})}), {x({"a","c"})}, + {x({"a","b"})}, float_cells({x({"a","c"})}), + float_cells({x({"a","b"})}), float_cells({x({"a","c"})}), + {x({"a","b","c"}),y({"d","e"})}, {x({"b","f"}),y({"d","g"})}, + {x(3),y({"a","b"})}, {x(3),y({"b","c"})} +}; + +TensorSpec reference_add(const TensorSpec &a, const TensorSpec &b) { + TensorSpec result(a.type()); + for (const auto &cell: b.cells()) { + result.add(cell.first, cell.second); + } + auto end_iter = b.cells().end(); + for (const auto &cell: a.cells()) { + auto iter = b.cells().find(cell.first); + if (iter == end_iter) { + result.add(cell.first, cell.second); + } + } + return result; +} + +Value::UP try_partial_add(const TensorSpec &a, const TensorSpec &b) { + const auto &factory = SimpleValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + return tensor::TensorPartialUpdate::add(*lhs, *rhs, factory); +} + +TensorSpec perform_partial_add(const TensorSpec &a, const TensorSpec &b) { + auto up = try_partial_add(a, b); + EXPECT_TRUE(up); + return spec_from_value(*up); +} + +TensorSpec perform_old_add(const TensorSpec &a, const TensorSpec &b) { + const auto &engine = tensor::DefaultTensorEngine::ref(); + auto lhs = engine.from_spec(a); + auto rhs = engine.from_spec(b); + auto lhs_tensor = dynamic_cast<tensor::Tensor *>(lhs.get()); + EXPECT_TRUE(lhs_tensor); + auto rhs_tensor = dynamic_cast<tensor::Tensor *>(rhs.get()); + EXPECT_TRUE(rhs_tensor); + auto up = lhs_tensor->add(*rhs_tensor); + EXPECT_TRUE(up); + return engine.to_spec(*up); +} + + +TEST(PartialAddTest, partial_add_works_for_simple_values) { + ASSERT_TRUE((add_layouts.size() % 2) == 0); + for (size_t i = 0; i < add_layouts.size(); i += 2) { + TensorSpec lhs = spec(add_layouts[i], N()); + TensorSpec rhs = spec(add_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = reference_add(lhs, rhs); + auto actual = perform_partial_add(lhs, rhs); + EXPECT_EQ(actual, expect); + } +} + +TEST(PartialAddTest, partial_add_works_like_old_add) { + ASSERT_TRUE((add_layouts.size() % 2) == 0); + for (size_t i = 0; i < add_layouts.size(); i += 2) { + TensorSpec lhs = spec(add_layouts[i], N()); + TensorSpec rhs = spec(add_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = perform_old_add(lhs, rhs); + auto actual = perform_partial_add(lhs, rhs); + EXPECT_EQ(actual, expect); + } +} + +std::vector<Layout> bad_layouts = { + {x(3)}, {x(3),y(1)}, + {x(3),y(1)}, {x(3)}, + {x(3),y(3)}, {x(3),y({"a"})}, + {x(3),y({"a"})}, {x(3),y(3)}, + {x({"a"})}, {x({"a"}),y({"b"})}, + {x({"a"}),y({"b"})}, {x({"a"})}, + {x({"a"})}, {x({"a"}),y(1)} +}; + +TEST(PartialAddTest, partial_add_returns_nullptr_on_invalid_inputs) { + ASSERT_TRUE((bad_layouts.size() % 2) == 0); + for (size_t i = 0; i < bad_layouts.size(); i += 2) { + TensorSpec lhs = spec(bad_layouts[i], N()); + TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto actual = try_partial_add(lhs, rhs); + auto expect = Value::UP(); + EXPECT_EQ(actual, expect); + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/partial_modify/CMakeLists.txt b/eval/src/tests/tensor/partial_modify/CMakeLists.txt new file mode 100644 index 00000000000..42a08acaae6 --- /dev/null +++ b/eval/src/tests/tensor/partial_modify/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_partial_modify_test_app TEST + SOURCES + partial_modify_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_partial_modify_test_app COMMAND eval_partial_modify_test_app) diff --git a/eval/src/tests/tensor/partial_modify/partial_modify_test.cpp b/eval/src/tests/tensor/partial_modify/partial_modify_test.cpp new file mode 100644 index 00000000000..677d6d71ee1 --- /dev/null +++ b/eval/src/tests/tensor/partial_modify/partial_modify_test.cpp @@ -0,0 +1,143 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/tensor/cell_values.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/partial_update.h> +#include <vespa/eval/tensor/sparse/sparse_tensor.h> +#include <vespa/eval/tensor/tensor.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <optional> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector<Layout> modify_layouts = { + {x({"a"})}, {x({"a"})}, + {x({"a",""})}, {x({"b","c","d","e"})}, + {x(5)}, {x({"1","2","foo","17"})}, + {x({"a","b","c"}),y({"d","e"})}, {x({"b"}),y({"d"})}, + {x({"a","b","c"})}, {x({"b","c","d"})}, + {x(4),y({"a","b","c","d"}),z(5)}, {x({"1","2"}),y({"b","d"}),z({"1","3"})}, + {x(3),y(2)}, {x({"0","1"}),y({"0","1"})}, + {x({"a","","b"})}, {x({""})} +}; + +TensorSpec::Address sparsify(const TensorSpec::Address &input) { + TensorSpec::Address output; + for (const auto & kv : input) { + if (kv.second.is_indexed()) { + auto val = fmt("%zu", kv.second.index); + output.emplace(kv.first, val); + } else { + output.emplace(kv.first, kv.second); + } + } + return output; +} + +TensorSpec reference_modify(const TensorSpec &a, const TensorSpec &b, join_fun_t fun) { + TensorSpec result(a.type()); + auto end_iter = b.cells().end(); + for (const auto &cell: a.cells()) { + double v = cell.second; + auto sparse_addr = sparsify(cell.first); + auto iter = b.cells().find(sparse_addr); + if (iter == end_iter) { + result.add(cell.first, v); + } else { + result.add(cell.first, fun(v, iter->second)); + } + } + return result; +} + +Value::UP try_partial_modify(const TensorSpec &a, const TensorSpec &b, join_fun_t fun) { + const auto &factory = SimpleValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + return tensor::TensorPartialUpdate::modify(*lhs, fun, *rhs, factory); +} + +TensorSpec perform_partial_modify(const TensorSpec &a, const TensorSpec &b, join_fun_t fun) { + auto up = try_partial_modify(a, b, fun); + EXPECT_TRUE(up); + return spec_from_value(*up); +} + +TensorSpec perform_old_modify(const TensorSpec &a, const TensorSpec &b, join_fun_t fun) { + const auto &engine = tensor::DefaultTensorEngine::ref(); + auto lhs = engine.from_spec(a); + auto rhs = engine.from_spec(b); + auto lhs_tensor = dynamic_cast<tensor::Tensor *>(lhs.get()); + EXPECT_TRUE(lhs_tensor); + auto rhs_sparse = dynamic_cast<tensor::SparseTensor *>(rhs.get()); + EXPECT_TRUE(rhs_sparse); + tensor::CellValues cell_values(*rhs_sparse); + auto up = lhs_tensor->modify(fun, cell_values); + EXPECT_TRUE(up); + return engine.to_spec(*up); +} + + +TEST(PartialModifyTest, partial_modify_works_for_simple_values) { + ASSERT_TRUE((modify_layouts.size() % 2) == 0); + for (size_t i = 0; i < modify_layouts.size(); i += 2) { + TensorSpec lhs = spec(modify_layouts[i], N()); + TensorSpec rhs = spec(modify_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f}) { + auto expect = reference_modify(lhs, rhs, fun); + auto actual = perform_partial_modify(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + auto fun = [](double, double keep) { return keep; }; + auto expect = reference_modify(lhs, rhs, fun); + auto actual = perform_partial_modify(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } +} + +TEST(PartialModifyTest, partial_modify_works_like_old_modify) { + ASSERT_TRUE((modify_layouts.size() % 2) == 0); + for (size_t i = 0; i < modify_layouts.size(); i += 2) { + TensorSpec lhs = spec(modify_layouts[i], N()); + TensorSpec rhs = spec(modify_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f}) { + auto expect = perform_old_modify(lhs, rhs, fun); + auto actual = perform_partial_modify(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + } +} + +std::vector<Layout> bad_layouts = { + {x(3)}, {x(3)}, + {x(3),y({"a"})}, {x(3),y({"a"})}, + {x({"a"})}, {x({"a"}),y({"b"})}, + {x({"a"}),y({"b"})}, {x({"a"})}, + {x({"a"})}, {x({"a"}),y(1)} +}; + +TEST(PartialModifyTest, partial_modify_returns_nullptr_on_invalid_inputs) { + ASSERT_TRUE((bad_layouts.size() % 2) == 0); + for (size_t i = 0; i < bad_layouts.size(); i += 2) { + TensorSpec lhs = spec(bad_layouts[i], N()); + TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + for (auto fun: {operation::Add::f}) { + auto actual = try_partial_modify(lhs, rhs, fun); + auto expect = Value::UP(); + EXPECT_EQ(actual, expect); + } + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/partial_remove/CMakeLists.txt b/eval/src/tests/tensor/partial_remove/CMakeLists.txt new file mode 100644 index 00000000000..1680324f574 --- /dev/null +++ b/eval/src/tests/tensor/partial_remove/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_partial_remove_test_app TEST + SOURCES + partial_remove_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_partial_remove_test_app COMMAND eval_partial_remove_test_app) diff --git a/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp b/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp new file mode 100644 index 00000000000..7d1611b7f64 --- /dev/null +++ b/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp @@ -0,0 +1,126 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/tensor/cell_values.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/partial_update.h> +#include <vespa/eval/tensor/sparse/sparse_tensor.h> +#include <vespa/eval/tensor/tensor.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <optional> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector<Layout> remove_layouts = { + {x({"a"})}, {x({"b"})}, + {x({"a","b"})}, {x({"a","c"})}, + {x({"a","b"})}, {x({"a","b"})}, + float_cells({x({"a","b"})}), {x({"a","c"})}, + {x({"a","b"})}, float_cells({x({"a","c"})}), + float_cells({x({"a","b"})}), float_cells({x({"a","c"})}), + {x({"a","b","c"}),y({"d","e"})}, {x({"b","f"}),y({"d","g"})}, + {x(3),y({"a","b"})}, {y({"b","c"})} +}; + +TensorSpec::Address only_sparse(const TensorSpec::Address &input) { + TensorSpec::Address output; + for (const auto & kv : input) { + if (kv.second.is_mapped()) { + output.emplace(kv.first, kv.second); + } + } + return output; +} + +TensorSpec reference_remove(const TensorSpec &a, const TensorSpec &b) { + TensorSpec result(a.type()); + auto end_iter = b.cells().end(); + for (const auto &cell: a.cells()) { + auto iter = b.cells().find(only_sparse(cell.first)); + if (iter == end_iter) { + result.add(cell.first, cell.second); + } + } + return result; +} + +Value::UP try_partial_remove(const TensorSpec &a, const TensorSpec &b) { + const auto &factory = SimpleValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + return tensor::TensorPartialUpdate::remove(*lhs, *rhs, factory); +} + +TensorSpec perform_partial_remove(const TensorSpec &a, const TensorSpec &b) { + auto up = try_partial_remove(a, b); + EXPECT_TRUE(up); + return spec_from_value(*up); +} + +TensorSpec perform_old_remove(const TensorSpec &a, const TensorSpec &b) { + const auto &engine = tensor::DefaultTensorEngine::ref(); + auto lhs = engine.from_spec(a); + auto rhs = engine.from_spec(b); + auto lhs_tensor = dynamic_cast<tensor::Tensor *>(lhs.get()); + EXPECT_TRUE(lhs_tensor); + auto rhs_sparse = dynamic_cast<tensor::SparseTensor *>(rhs.get()); + EXPECT_TRUE(rhs_sparse); + tensor::CellValues cell_values(*rhs_sparse); + auto up = lhs_tensor->remove(cell_values); + EXPECT_TRUE(up); + return engine.to_spec(*up); +} + + +TEST(PartialRemoveTest, partial_remove_works_for_simple_values) { + ASSERT_TRUE((remove_layouts.size() % 2) == 0); + for (size_t i = 0; i < remove_layouts.size(); i += 2) { + TensorSpec lhs = spec(remove_layouts[i], N()); + TensorSpec rhs = spec(remove_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = reference_remove(lhs, rhs); + auto actual = perform_partial_remove(lhs, rhs); + EXPECT_EQ(actual, expect); + } +} + +TEST(PartialRemoveTest, partial_remove_works_like_old_remove) { + ASSERT_TRUE((remove_layouts.size() % 2) == 0); + for (size_t i = 0; i < remove_layouts.size(); i += 2) { + TensorSpec lhs = spec(remove_layouts[i], N()); + TensorSpec rhs = spec(remove_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = perform_old_remove(lhs, rhs); + auto actual = perform_partial_remove(lhs, rhs); + EXPECT_EQ(actual, expect); + } +} + +std::vector<Layout> bad_layouts = { + {x(3)}, {x(3)}, + {x(3),y({"a"})}, {x(3)}, + {x(3),y({"a"})}, {x(3),y({"a"})}, + {x({"a"})}, {y({"a"})}, + {x({"a"})}, {x({"a"}),y({"b"})} +}; + +TEST(PartialRemoveTest, partial_remove_returns_nullptr_on_invalid_inputs) { + ASSERT_TRUE((bad_layouts.size() % 2) == 0); + for (size_t i = 0; i < bad_layouts.size(); i += 2) { + TensorSpec lhs = spec(bad_layouts[i], N()); + TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N())); + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto actual = try_partial_remove(lhs, rhs); + auto expect = Value::UP(); + EXPECT_EQ(actual, expect); + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index e1009969b43..358f5d36101 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -11,6 +11,7 @@ #include <ostream> #include <vespa/eval/tensor/dense/dense_tensor_view.h> #include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/eval/simple_value.h> using namespace vespalib::tensor; using vespalib::eval::TensorSpec; diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index e2bcee94498..244966fc992 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -8,6 +8,8 @@ vespa_add_library(eval_eval OBJECT delete_node.cpp double_value_builder.cpp fast_forest.cpp + fast_sparse_map.cpp + fast_value.cpp function.cpp gbdt.cpp interpreted_function.cpp @@ -19,7 +21,6 @@ vespa_add_library(eval_eval OBJECT operation.cpp operator_nodes.cpp param_usage.cpp - simple_sparse_map.cpp simple_tensor.cpp simple_tensor_engine.cpp simple_value.cpp diff --git a/eval/src/vespa/eval/eval/fast_sparse_map.cpp b/eval/src/vespa/eval/eval/fast_sparse_map.cpp new file mode 100644 index 00000000000..3215b5f8995 --- /dev/null +++ b/eval/src/vespa/eval/eval/fast_sparse_map.cpp @@ -0,0 +1,12 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "fast_sparse_map.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + +namespace vespalib::eval { + +FastSparseMap::~FastSparseMap() = default; + +} + +VESPALIB_HASH_MAP_INSTANTIATE_H_E(vespalib::eval::FastSparseMap::Key, uint32_t, vespalib::eval::FastSparseMap::Hash, vespalib::eval::FastSparseMap::Equal); diff --git a/eval/src/vespa/eval/eval/fast_sparse_map.h b/eval/src/vespa/eval/eval/fast_sparse_map.h new file mode 100644 index 00000000000..666a9db4140 --- /dev/null +++ b/eval/src/vespa/eval/eval/fast_sparse_map.h @@ -0,0 +1,153 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/arrayref.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vector> +#include <xxhash.h> +#include <type_traits> + +namespace vespalib::eval { + +/** + * A wrapper around vespalib::hash_map, using it to map a list of + * labels (a sparse address) to an integer value (dense subspace + * index). Labels are stored in a separate vector to avoid + * fragmentation caused by hash keys being vectors of values. Labels + * can be specified in different ways during lookup and insert in + * order to reduce the need for data restructuring when used to + * integrate with the Value api. All labels are stored with a 64-bit + * hash. This hash is used as label equality (assuming no + * collisions). A order-sensitive 64bit hash constructed from + * individual label hashes is used for address equality (also assuming + * no collisions). The hash algorithm currently used is XXH3. + * + * 'add_mapping' will will bind the given address to an integer value + * equal to the current (pre-insert) size of the map. The given + * address MUST NOT already be in the map. + * + * 'lookup' will return the integer value associated with the + * given address or a special npos value if the value is not found. + **/ +class FastSparseMap +{ +public: + static uint64_t hash_label(const vespalib::string &str) { + return XXH3_64bits(str.data(), str.size()); + } + static uint64_t hash_label(vespalib::stringref str) { + return XXH3_64bits(str.data(), str.size()); + } + static uint64_t hash_label(const vespalib::stringref *str) { + return XXH3_64bits(str->data(), str->size()); + } + + struct HashedLabel { + vespalib::string label; + uint64_t hash; + HashedLabel() : label(), hash(0) {} + HashedLabel(const HashedLabel &rhs) = default; + HashedLabel &operator=(const HashedLabel &rhs) = default; + HashedLabel(HashedLabel &&rhs) = default; + HashedLabel &operator=(HashedLabel &&rhs) = default; + HashedLabel(const vespalib::string &str) : label(str), hash(hash_label(str)) {} + HashedLabel(vespalib::stringref str) : label(str), hash(hash_label(str)) {} + HashedLabel(const vespalib::stringref *str) : label(*str), hash(hash_label(*str)) {} + }; + + static uint64_t hash_label(const HashedLabel &label) { + return label.hash; + } + + struct Key { + uint64_t hash; + Key() : hash(0) {} + Key(uint64_t hash_in) + : hash(hash_in) {} + } __attribute__((packed,aligned(4))); + + struct Hash { + uint64_t operator()(const Key &key) const { return key.hash; } + uint64_t operator()(uint64_t hash) const { return hash; } + }; + + struct Equal { + bool operator()(const Key &a, uint64_t b) const { return (a.hash == b); } + bool operator()(const Key &a, const Key &b) const { return (a.hash == b.hash); } + }; + + using MapType = vespalib::hash_map<Key,uint32_t,Hash,Equal>; + +private: + size_t _num_dims; + std::vector<HashedLabel> _labels; + MapType _map; + +public: + FastSparseMap(size_t num_dims_in, size_t expected_subspaces) + : _num_dims(num_dims_in), _labels(), _map(expected_subspaces * 2) + { + static_assert(std::is_same_v<XXH64_hash_t, uint64_t>); + _labels.reserve(_num_dims * expected_subspaces); + } + ~FastSparseMap(); + size_t size() const { return _map.size(); } + size_t num_dims() const { return _num_dims; } + static constexpr size_t npos() { return -1; } + const std::vector<HashedLabel> &labels() const { return _labels; } + + ConstArrayRef<HashedLabel> make_addr(uint32_t index) const { + return ConstArrayRef<HashedLabel>(&_labels[index * _num_dims], _num_dims); + } + + template <typename T> + uint64_t hash_addr(ConstArrayRef<T> addr) const { + uint64_t h = 0; + for (const auto &label: addr) { + h = 31 * h + hash_label(label); + } + return h; + } + + template <typename T> + void add_mapping(ConstArrayRef<T> addr, uint64_t hash) { + uint32_t value = _map.size(); + for (const auto &label: addr) { + _labels.emplace_back(label); + } + _map.insert(std::make_pair(Key(hash), value)); + } + + template <typename T> + void add_mapping(ConstArrayRef<T> addr) { + uint64_t h = 0; + uint32_t value = _map.size(); + for (const auto &label: addr) { + _labels.emplace_back(label); + h = 31 * h + hash_label(_labels.back()); + } + _map.insert(std::make_pair(Key(h), value)); + } + + size_t lookup(uint64_t hash) const { + auto pos = _map.find(hash); + return (pos == _map.end()) ? npos() : pos->second; + } + + template <typename T> + size_t lookup(ConstArrayRef<T> addr) const { + return lookup(hash_addr(addr)); + } + + template <typename F> + void each_map_entry(F &&f) const { + _map.for_each([&](const auto &entry) + { + f(entry.second, entry.first.hash); + }); + } +}; + +} diff --git a/eval/src/vespa/eval/eval/fast_value.cpp b/eval/src/vespa/eval/eval/fast_value.cpp new file mode 100644 index 00000000000..2e8226257a4 --- /dev/null +++ b/eval/src/vespa/eval/eval/fast_value.cpp @@ -0,0 +1,54 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "fast_value.h" +#include <vespa/vespalib/util/typify.h> +#include "fast_value.hpp" + +namespace vespalib::eval { + +//----------------------------------------------------------------------------- + +namespace { + +struct CreateFastValueBuilderBase { + template <typename T> static std::unique_ptr<ValueBuilderBase> invoke(const ValueType &type, + size_t num_mapped_dims, size_t subspace_size, size_t expected_subspaces) + { + assert(check_cell_type<T>(type.cell_type())); + return std::make_unique<FastValue<T>>(type, num_mapped_dims, subspace_size, expected_subspaces); + } +}; + +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + +std::unique_ptr<Value::Index::View> +FastValueIndex::create_view(const std::vector<size_t> &dims) const +{ + if (map.num_dims() == 0) { + return TrivialIndex::get().create_view(dims); + } else if (dims.empty()) { + return std::make_unique<IterateView>(map); + } else if (dims.size() == map.num_dims()) { + return std::make_unique<LookupView>(map); + } else { + return std::make_unique<FilterView>(map, dims); + } +} + +//----------------------------------------------------------------------------- + +FastValueBuilderFactory::FastValueBuilderFactory() = default; +FastValueBuilderFactory FastValueBuilderFactory::_factory; + +std::unique_ptr<ValueBuilderBase> +FastValueBuilderFactory::create_value_builder_base(const ValueType &type, size_t num_mapped_dims, size_t subspace_size, + size_t expected_subspaces) const +{ + return typify_invoke<1,TypifyCellType,CreateFastValueBuilderBase>(type.cell_type(), type, num_mapped_dims, subspace_size, expected_subspaces); +} + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/eval/fast_value.h b/eval/src/vespa/eval/eval/fast_value.h new file mode 100644 index 00000000000..ac924ecc6eb --- /dev/null +++ b/eval/src/vespa/eval/eval/fast_value.h @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value.h" + +namespace vespalib::eval { + +/** + * A fast value is a value that uses a FastValueIndex to store its + * sparse mappings. The FastValueIndex class contains inlined + * functions that can be called directly from instruction + * implementations to speed up sparse operations. Also, the + * FastSparseMap used by the FastValueIndex is a highly optimized + * alternative to the map used by SimpleValue, which means that normal + * Value API usage will also have improved performance. + **/ +class FastValueBuilderFactory : public ValueBuilderFactory { +private: + FastValueBuilderFactory(); + static FastValueBuilderFactory _factory; + std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, + size_t num_mapped_dims, size_t subspace_size, size_t expected_subspaces) const override; +public: + static const FastValueBuilderFactory &get() { return _factory; } +}; + +} diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp new file mode 100644 index 00000000000..4efb56ee0c4 --- /dev/null +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -0,0 +1,215 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "value.h" +#include "fast_sparse_map.h" +#include "inline_operation.h" +#include <vespa/eval/instruction/generic_join.h> +#include <vespa/vespalib/stllike/hash_map.hpp> + +namespace vespalib::eval { + +//----------------------------------------------------------------------------- + +namespace { + +//----------------------------------------------------------------------------- + +// look up a full address in the map directly +struct LookupView : public Value::Index::View { + + const FastSparseMap ↦ + size_t subspace; + + LookupView(const FastSparseMap &map_in) + : map(map_in), subspace(FastSparseMap::npos()) {} + + void lookup(ConstArrayRef<const vespalib::stringref*> addr) override { + subspace = map.lookup(addr); + } + + bool next_result(ConstArrayRef<vespalib::stringref*>, size_t &idx_out) override { + if (subspace == FastSparseMap::npos()) { + return false; + } + idx_out = subspace; + subspace = FastSparseMap::npos(); + return true; + } +}; + +//----------------------------------------------------------------------------- + +// find matching mappings for a partial address with brute force filtering +struct FilterView : public Value::Index::View { + + using Label = FastSparseMap::HashedLabel; + + size_t num_mapped_dims; + const std::vector<Label> &labels; + std::vector<size_t> match_dims; + std::vector<size_t> extract_dims; + std::vector<Label> query; + size_t pos; + + bool is_match() const { + for (size_t i = 0; i < query.size(); ++i) { + if (query[i].hash != labels[pos + match_dims[i]].hash) { + return false; + } + } + return true; + } + + FilterView(const FastSparseMap &map, const std::vector<size_t> &match_dims_in) + : num_mapped_dims(map.num_dims()), labels(map.labels()), match_dims(match_dims_in), + extract_dims(), query(match_dims.size(), Label()), pos(labels.size()) + { + auto my_pos = match_dims.begin(); + for (size_t i = 0; i < num_mapped_dims; ++i) { + if ((my_pos == match_dims.end()) || (*my_pos != i)) { + extract_dims.push_back(i); + } else { + ++my_pos; + } + } + assert(my_pos == match_dims.end()); + assert((match_dims.size() + extract_dims.size()) == num_mapped_dims); + } + + void lookup(ConstArrayRef<const vespalib::stringref*> addr) override { + assert(addr.size() == query.size()); + for (size_t i = 0; i < addr.size(); ++i) { + query[i] = Label(*addr[i]); + } + pos = 0; + } + + bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override { + while (pos < labels.size()) { + if (is_match()) { + assert(addr_out.size() == extract_dims.size()); + for (size_t i = 0; i < extract_dims.size(); ++i) { + *addr_out[i] = labels[pos + extract_dims[i]].label; + } + idx_out = (pos / num_mapped_dims); // is this expensive? + pos += num_mapped_dims; + return true; + } + pos += num_mapped_dims; + } + return false; + } +}; + +//----------------------------------------------------------------------------- + +// iterate all mappings +struct IterateView : public Value::Index::View { + + using Labels = std::vector<FastSparseMap::HashedLabel>; + + size_t num_mapped_dims; + const Labels &labels; + size_t pos; + + IterateView(const FastSparseMap &map) + : num_mapped_dims(map.num_dims()), labels(map.labels()), pos(labels.size()) {} + + void lookup(ConstArrayRef<const vespalib::stringref*>) override { + pos = 0; + } + + bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override { + if (pos >= labels.size()) { + return false; + } + assert(addr_out.size() == num_mapped_dims); + for (size_t i = 0; i < num_mapped_dims; ++i) { + *addr_out[i] = labels[pos + i].label; + } + idx_out = (pos / num_mapped_dims); // is this expensive? + pos += num_mapped_dims; + return true; + } +}; + +//----------------------------------------------------------------------------- + +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + +// This is the class instructions will look for when optimizing sparse +// operations by calling inline functions directly. + +struct FastValueIndex final : Value::Index { + FastSparseMap map; + FastValueIndex(size_t num_mapped_dims_in, size_t expected_subspaces_in) + : map(num_mapped_dims_in, expected_subspaces_in) {} + + template <typename LCT, typename RCT, typename OCT, typename Fun> + static const Value &sparse_full_overlap_join(const ValueType &res_type, const Fun &fun, + const FastValueIndex &lhs, const FastValueIndex &rhs, + ConstArrayRef<LCT> lhs_cells, ConstArrayRef<RCT> rhs_cells, Stash &stash); + + size_t size() const override { return map.size(); } + std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; +}; + +//----------------------------------------------------------------------------- + +template <typename T> +struct FastValue final : Value, ValueBuilder<T> { + + ValueType my_type; + size_t my_subspace_size; + FastValueIndex my_index; + std::vector<T> my_cells; + + FastValue(const ValueType &type_in, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces_in) + : my_type(type_in), my_subspace_size(subspace_size_in), my_index(num_mapped_dims_in, expected_subspaces_in), my_cells() + { + my_cells.reserve(subspace_size_in * expected_subspaces_in); + } + ~FastValue() override; + const ValueType &type() const override { return my_type; } + const Value::Index &index() const override { return my_index; } + TypedCells cells() const override { return TypedCells(ConstArrayRef<T>(my_cells)); } + ArrayRef<T> add_subspace(ConstArrayRef<vespalib::stringref> addr) override { + size_t old_size = my_cells.size(); + my_index.map.add_mapping(addr); + my_cells.resize(old_size + my_subspace_size); + return ArrayRef<T>(&my_cells[old_size], my_subspace_size); + } + std::unique_ptr<Value> build(std::unique_ptr<ValueBuilder<T>> self) override { + ValueBuilder<T>* me = this; + assert(me == self.get()); + self.release(); + return std::unique_ptr<Value>(this); + } +}; +template <typename T> FastValue<T>::~FastValue() = default; + +//----------------------------------------------------------------------------- + +template <typename LCT, typename RCT, typename OCT, typename Fun> +const Value & +FastValueIndex::sparse_full_overlap_join(const ValueType &res_type, const Fun &fun, + const FastValueIndex &lhs, const FastValueIndex &rhs, + ConstArrayRef<LCT> lhs_cells, ConstArrayRef<RCT> rhs_cells, Stash &stash) +{ + auto &result = stash.create<FastValue<OCT>>(res_type, lhs.map.num_dims(), 1, lhs.map.size()); + lhs.map.each_map_entry([&](auto lhs_subspace, auto hash) + { + auto rhs_subspace = rhs.map.lookup(hash); + if (rhs_subspace != FastSparseMap::npos()) { + result.my_index.map.add_mapping(lhs.map.make_addr(lhs_subspace), hash); + result.my_cells.push_back(fun(lhs_cells[lhs_subspace], rhs_cells[rhs_subspace])); + } + }); + return result; +} + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/eval/simple_sparse_map.cpp b/eval/src/vespa/eval/eval/simple_sparse_map.cpp deleted file mode 100644 index 18d3bffd47d..00000000000 --- a/eval/src/vespa/eval/eval/simple_sparse_map.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simple_sparse_map.h" -#include <vespa/vespalib/stllike/hash_map.hpp> - -namespace vespalib::eval { - -SimpleSparseMap::~SimpleSparseMap() = default; - -} - -VESPALIB_HASH_MAP_INSTANTIATE_H_E(vespalib::eval::SimpleSparseMap::Key, uint32_t, vespalib::eval::SimpleSparseMap::Hash, vespalib::eval::SimpleSparseMap::Equal); diff --git a/eval/src/vespa/eval/eval/simple_sparse_map.h b/eval/src/vespa/eval/eval/simple_sparse_map.h deleted file mode 100644 index 61ff9f326e2..00000000000 --- a/eval/src/vespa/eval/eval/simple_sparse_map.h +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/util/arrayref.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/hash_map.h> -#include <vector> -#include <cassert> - -namespace vespalib::eval { - -/** - * A simple wrapper around vespalib::hash_map, using it to map a list - * of labels (a sparse address) to an integer value (dense subspace - * index). Labels are stored in a separate vector and the map keys - * reference a slice of this vector. This is to avoid fragmentation - * caused by hash keys being vectors of values. In addition, labels - * can be specified in different ways during lookup and insert in - * order to reduce the need for data restructuring when using the - * map. To keep things simple, map iterators are kept away from the - * api. This will have a minor overhead during lookup since the end - * iterator needs to be translated to npos. All added mappings are - * checked for uniqueness with an assert. There is no real need for - * map entry iteration since you can just iterate the labels vector - * directly. - * - * 'add_mapping' will will bind the given address to an integer value - * equal to the current (pre-insert) size of the map. The given - * address MUST NOT already be in the map. - * - * 'lookup' will return the integer value associated with the - * given address or a special npos value if the value is not found. - **/ -class SimpleSparseMap -{ -public: - using DirectStr = ConstArrayRef<vespalib::string>; - using DirectRef = ConstArrayRef<vespalib::stringref>; - using IndirectRef = ConstArrayRef<const vespalib::stringref *>; - - struct Key { - uint32_t start; - uint32_t end; - Key() : start(0), end(0) {} - Key(uint32_t start_in, uint32_t end_in) - : start(start_in), end(end_in) {} - }; - - struct Hash { - const std::vector<vespalib::string> *labels; - const vespalib::string &get_label(size_t i) const { return (*labels)[i]; } - Hash() : labels(nullptr) {} - Hash(const Hash &rhs) = default; - Hash &operator=(const Hash &rhs) = default; - Hash(const std::vector<vespalib::string> &labels_in) : labels(&labels_in) {} - size_t operator()(const Key &key) const { - size_t h = 0; - for (size_t i = key.start; i < key.end; ++i) { - const vespalib::string &str = get_label(i); - h = h * 31 + hashValue(str.data(), str.size()); - } - return h; - } - size_t operator()(const DirectStr &addr) const { - size_t h = 0; - for (const auto &str: addr) { - h = h * 31 + hashValue(str.data(), str.size()); - } - return h; - } - size_t operator()(const DirectRef &addr) const { - size_t h = 0; - for (const auto &str: addr) { - h = h * 31 + hashValue(str.data(), str.size()); - } - return h; - } - size_t operator()(const IndirectRef &addr) const { - size_t h = 0; - for (const auto *str: addr) { - h = h * 31 + hashValue(str->data(), str->size()); - } - return h; - } - }; - - struct Equal { - const std::vector<vespalib::string> *labels; - const vespalib::string &get_label(size_t i) const { return (*labels)[i]; } - Equal() : labels(nullptr) {} - Equal(const Equal &rhs) = default; - Equal &operator=(const Equal &rhs) = default; - Equal(const std::vector<vespalib::string> &labels_in) : labels(&labels_in) {} - bool operator()(const Key &a, const Key &b) const { - size_t len = (a.end - a.start); - if ((b.end - b.start) != len) { - return false; - } - for (size_t i = 0; i < len; ++i) { - if (get_label(a.start + i) != get_label(b.start + i)) { - return false; - } - } - return true; - } - bool operator()(const Key &a, const DirectStr &addr) const { - if (addr.size() != (a.end - a.start)) { - return false; - } - for (size_t i = 0; i < addr.size(); ++i) { - if (get_label(a.start + i) != addr[i]) { - return false; - } - } - return true; - } - bool operator()(const Key &a, const DirectRef &addr) const { - if (addr.size() != (a.end - a.start)) { - return false; - } - for (size_t i = 0; i < addr.size(); ++i) { - if (get_label(a.start + i) != addr[i]) { - return false; - } - } - return true; - } - bool operator()(const Key &a, const IndirectRef &addr) const { - if (addr.size() != (a.end - a.start)) { - return false; - } - for (size_t i = 0; i < addr.size(); ++i) { - if (get_label(a.start + i) != *addr[i]) { - return false; - } - } - return true; - } - }; - - using MapType = vespalib::hash_map<Key,uint32_t,Hash,Equal>; - -private: - std::vector<vespalib::string> _labels; - MapType _map; - -public: - SimpleSparseMap(size_t num_mapped_dims, size_t expected_subspaces) - : _labels(), _map(expected_subspaces * 2, Hash(_labels), Equal(_labels)) - { - _labels.reserve(num_mapped_dims * expected_subspaces); - } - ~SimpleSparseMap(); - size_t size() const { return _map.size(); } - static constexpr size_t npos() { return -1; } - const std::vector<vespalib::string> &labels() const { return _labels; } - void add_mapping(DirectStr addr) { - uint32_t value = _map.size(); - uint32_t start = _labels.size(); - for (const auto &label: addr) { - _labels.emplace_back(label); - } - uint32_t end = _labels.size(); - auto [ignore, was_inserted] = _map.insert(std::make_pair(Key(start, end), value)); - assert(was_inserted); - } - void add_mapping(DirectRef addr) { - uint32_t value = _map.size(); - uint32_t start = _labels.size(); - for (const auto &label: addr) { - _labels.emplace_back(label); - } - uint32_t end = _labels.size(); - auto [ignore, was_inserted] = _map.insert(std::make_pair(Key(start, end), value)); - assert(was_inserted); - } - void add_mapping(IndirectRef addr) { - uint32_t value = _map.size(); - uint32_t start = _labels.size(); - for (const auto *label: addr) { - _labels.emplace_back(*label); - } - uint32_t end = _labels.size(); - auto [ignore, was_inserted] = _map.insert(std::make_pair(Key(start, end), value)); - assert(was_inserted); - } - size_t lookup(DirectStr addr) const { - auto pos = _map.find(addr); - return (pos == _map.end()) ? npos() : pos->second; - } - size_t lookup(DirectRef addr) const { - auto pos = _map.find(addr); - return (pos == _map.end()) ? npos() : pos->second; - } - size_t lookup(IndirectRef addr) const { - auto pos = _map.find(addr); - return (pos == _map.end()) ? npos() : pos->second; - } -}; - -} diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index 84ba35404fd..cca4c6c7d51 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -30,22 +30,30 @@ struct CreateSimpleValueBuilderBase { // look up a full address in the map directly struct LookupView : public Value::Index::View { - const SimpleSparseMap &index; - size_t subspace; + using Labels = std::vector<vespalib::string>; + using Map = std::map<Labels, size_t>; - LookupView(const SimpleSparseMap &index_in) - : index(index_in), subspace(SimpleSparseMap::npos()) {} + const Map ↦ + Labels my_addr; + Map::const_iterator pos; + + LookupView(const Map &map_in, size_t num_dims) + : map(map_in), my_addr(num_dims, ""), pos(map.end()) {} void lookup(ConstArrayRef<const vespalib::stringref*> addr) override { - subspace = index.lookup(addr); + assert(addr.size() == my_addr.size()); + for (size_t i = 0; i < my_addr.size(); ++i) { + my_addr[i] = *addr[i]; + } + pos = map.find(my_addr); } bool next_result(ConstArrayRef<vespalib::stringref*>, size_t &idx_out) override { - if (subspace == SimpleSparseMap::npos()) { + if (pos == map.end()) { return false; } - idx_out = subspace; - subspace = SimpleSparseMap::npos(); + idx_out = pos->second; + pos = map.end(); return true; } }; @@ -55,28 +63,29 @@ struct LookupView : public Value::Index::View { // find matching mappings for a partial address with brute force filtering struct FilterView : public Value::Index::View { - size_t num_mapped_dims; - const std::vector<vespalib::string> &labels; - std::vector<size_t> match_dims; - std::vector<size_t> extract_dims; - std::vector<vespalib::string> query; - size_t pos; + using Labels = std::vector<vespalib::string>; + using Map = std::map<Labels, size_t>; + + const Map ↦ + std::vector<size_t> match_dims; + std::vector<size_t> extract_dims; + std::vector<vespalib::string> query; + Map::const_iterator pos; bool is_match() const { for (size_t i = 0; i < query.size(); ++i) { - if (query[i] != labels[pos + match_dims[i]]) { + if (query[i] != pos->first[match_dims[i]]) { return false; } } return true; } - FilterView(const std::vector<vespalib::string> &labels_in, const std::vector<size_t> &match_dims_in, size_t num_mapped_dims_in) - : num_mapped_dims(num_mapped_dims_in), labels(labels_in), match_dims(match_dims_in), - extract_dims(), query(match_dims.size(), ""), pos(labels.size()) + FilterView(const Map &map_in, const std::vector<size_t> &match_dims_in, size_t num_dims) + : map(map_in), match_dims(match_dims_in), extract_dims(), query(match_dims.size(), ""), pos(map.end()) { auto my_pos = match_dims.begin(); - for (size_t i = 0; i < num_mapped_dims; ++i) { + for (size_t i = 0; i < num_dims; ++i) { if ((my_pos == match_dims.end()) || (*my_pos != i)) { extract_dims.push_back(i); } else { @@ -84,7 +93,7 @@ struct FilterView : public Value::Index::View { } } assert(my_pos == match_dims.end()); - assert((match_dims.size() + extract_dims.size()) == num_mapped_dims); + assert((match_dims.size() + extract_dims.size()) == num_dims); } void lookup(ConstArrayRef<const vespalib::stringref*> addr) override { @@ -92,21 +101,21 @@ struct FilterView : public Value::Index::View { for (size_t i = 0; i < addr.size(); ++i) { query[i] = *addr[i]; } - pos = 0; + pos = map.begin(); } bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override { - while (pos < labels.size()) { + while (pos != map.end()) { if (is_match()) { assert(addr_out.size() == extract_dims.size()); for (size_t i = 0; i < extract_dims.size(); ++i) { - *addr_out[i] = labels[pos + extract_dims[i]]; + *addr_out[i] = pos->first[extract_dims[i]]; } - idx_out = (pos / num_mapped_dims); // is this expensive? - pos += num_mapped_dims; + idx_out = pos->second; + ++pos; return true; } - pos += num_mapped_dims; + ++pos; } return false; } @@ -117,27 +126,29 @@ struct FilterView : public Value::Index::View { // iterate all mappings struct IterateView : public Value::Index::View { - size_t num_mapped_dims; - const std::vector<vespalib::string> &labels; - size_t pos; + using Labels = std::vector<vespalib::string>; + using Map = std::map<Labels, size_t>; + + const Map ↦ + Map::const_iterator pos; - IterateView(const std::vector<vespalib::string> &labels_in, size_t num_mapped_dims_in) - : num_mapped_dims(num_mapped_dims_in), labels(labels_in), pos(labels.size()) {} + IterateView(const Map &map_in) + : map(map_in), pos(map.end()) {} void lookup(ConstArrayRef<const vespalib::stringref*>) override { - pos = 0; + pos = map.begin(); } bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override { - if (pos >= labels.size()) { + if (pos == map.end()) { return false; } - assert(addr_out.size() == num_mapped_dims); - for (size_t i = 0; i < num_mapped_dims; ++i) { - *addr_out[i] = labels[pos + i]; + assert(addr_out.size() == pos->first.size()); + for (size_t i = 0; i < addr_out.size(); ++i) { + *addr_out[i] = pos->first[i]; } - idx_out = (pos / num_mapped_dims); // is this expensive? - pos += num_mapped_dims; + idx_out = pos->second; + ++pos; return true; } }; @@ -148,11 +159,11 @@ struct IterateView : public Value::Index::View { //----------------------------------------------------------------------------- -SimpleValue::SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces_in) +SimpleValue::SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in) : _type(type), _num_mapped_dims(num_mapped_dims_in), _subspace_size(subspace_size_in), - _index(num_mapped_dims_in, expected_subspaces_in) + _index() { assert(_type.count_mapped_dimensions() == _num_mapped_dims); assert(_type.dense_subspace_size() == _subspace_size); @@ -160,17 +171,26 @@ SimpleValue::SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_ SimpleValue::~SimpleValue() = default; +void +SimpleValue::add_mapping(ConstArrayRef<vespalib::stringref> addr) +{ + Labels my_addr; + for(const auto &label: addr) { + my_addr.emplace_back(label); + } + auto [ignore, was_inserted] = _index.emplace(my_addr, _index.size()); + assert(was_inserted); +} + std::unique_ptr<Value::Index::View> SimpleValue::create_view(const std::vector<size_t> &dims) const { - if (_num_mapped_dims == 0) { - return TrivialIndex::get().create_view(dims); - } else if (dims.empty()) { - return std::make_unique<IterateView>(_index.labels(), _num_mapped_dims); + if (dims.empty()) { + return std::make_unique<IterateView>(_index); } else if (dims.size() == _num_mapped_dims) { - return std::make_unique<LookupView>(_index); + return std::make_unique<LookupView>(_index, _num_mapped_dims); } else { - return std::make_unique<FilterView>(_index.labels(), dims, _num_mapped_dims); + return std::make_unique<FilterView>(_index, dims, _num_mapped_dims); } } @@ -178,7 +198,7 @@ SimpleValue::create_view(const std::vector<size_t> &dims) const template <typename T> SimpleValueT<T>::SimpleValueT(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces_in) - : SimpleValue(type, num_mapped_dims_in, subspace_size_in, expected_subspaces_in), + : SimpleValue(type, num_mapped_dims_in, subspace_size_in), _cells() { _cells.reserve(subspace_size_in * expected_subspaces_in); @@ -194,6 +214,7 @@ SimpleValueT<T>::add_subspace(ConstArrayRef<vespalib::stringref> addr) size_t old_size = _cells.size(); add_mapping(addr); _cells.resize(old_size + subspace_size()); + assert(_cells.size() == (size() * subspace_size())); return ArrayRef<T>(&_cells[old_size], subspace_size()); } diff --git a/eval/src/vespa/eval/eval/simple_value.h b/eval/src/vespa/eval/eval/simple_value.h index f2df087cf37..81f65eee061 100644 --- a/eval/src/vespa/eval/eval/simple_value.h +++ b/eval/src/vespa/eval/eval/simple_value.h @@ -3,7 +3,6 @@ #pragma once #include "value.h" -#include "simple_sparse_map.h" #include <vespa/vespalib/stllike/string.h> #include <vector> #include <map> @@ -12,27 +11,27 @@ namespace vespalib { class Stash; } namespace vespalib::eval { -class TensorSpec; - /** * A simple implementation of a generic value that can also be used to - * build new values. This class focuses on simplicity and is intended - * as a reference implementation that can also be used to test the - * correctness of tensor operations as they are moved away from the - * implementation of individual tensor classes. + * build new values. This class focuses on simplicity over speed and + * is intended as a reference implementation that can also be used to + * test the correctness of tensor operations as they are moved away + * from the implementation of individual tensor classes. **/ class SimpleValue : public Value, public Value::Index { private: + using Labels = std::vector<vespalib::string>; + ValueType _type; size_t _num_mapped_dims; size_t _subspace_size; - SimpleSparseMap _index; + std::map<Labels,size_t> _index; protected: size_t subspace_size() const { return _subspace_size; } - void add_mapping(ConstArrayRef<vespalib::stringref> addr) { _index.add_mapping(addr); } + void add_mapping(ConstArrayRef<vespalib::stringref> addr); public: - SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces_in); + SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in); ~SimpleValue() override; const ValueType &type() const override { return _type; } const Value::Index &index() const override { return *this; } diff --git a/eval/src/vespa/eval/eval/tensor_spec.cpp b/eval/src/vespa/eval/eval/tensor_spec.cpp index d3aafc7632a..1a6a9520e58 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.cpp +++ b/eval/src/vespa/eval/eval/tensor_spec.cpp @@ -46,9 +46,14 @@ TensorSpec & TensorSpec::operator = (const TensorSpec &) = default; TensorSpec::~TensorSpec() { } TensorSpec & -TensorSpec::set(Address address, double value) { - auto res = _cells.emplace(std::move(address), value); - if (!res.second) { assert(res.first->second.value == value); } +TensorSpec::add(Address address, double value) { + auto [iter, inserted] = _cells.emplace(std::move(address), value); + if (! inserted) { + // to simplify reference implementations, allow + // adding the same address several times to a Spec, but + // only with the same value every time: + assert(iter->second.value == value); + } return *this; } diff --git a/eval/src/vespa/eval/eval/tensor_spec.h b/eval/src/vespa/eval/eval/tensor_spec.h index 8a4343b3faa..8f02e56f860 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.h +++ b/eval/src/vespa/eval/eval/tensor_spec.h @@ -68,14 +68,7 @@ public: TensorSpec(const TensorSpec &); TensorSpec & operator = (const TensorSpec &); ~TensorSpec(); - TensorSpec &set(Address address, double value); - TensorSpec &add(Address address, double value) { - auto res = _cells.emplace(std::move(address), value); - if (!res.second) { - res.first->second.value += value; - } - return *this; - } + TensorSpec &add(Address address, double value); const vespalib::string &type() const { return _type; } const Cells &cells() const { return _cells; } vespalib::string to_string() const; diff --git a/eval/src/vespa/eval/eval/value_codec.h b/eval/src/vespa/eval/eval/value_codec.h index 3644f952a6c..058b2d7bf4f 100644 --- a/eval/src/vespa/eval/eval/value_codec.h +++ b/eval/src/vespa/eval/eval/value_codec.h @@ -2,7 +2,8 @@ #pragma once -#include "simple_value.h" +#include "value.h" +#include "tensor_spec.h" #include <vespa/vespalib/stllike/string.h> namespace vespalib { class nbostream; } diff --git a/eval/src/vespa/eval/instruction/generic_concat.cpp b/eval/src/vespa/eval/instruction/generic_concat.cpp index e3b3a3f0331..7c55afafcc1 100644 --- a/eval/src/vespa/eval/instruction/generic_concat.cpp +++ b/eval/src/vespa/eval/instruction/generic_concat.cpp @@ -3,6 +3,7 @@ #include "generic_concat.h" #include "generic_join.h" #include <vespa/eval/eval/value.h> +#include <vespa/eval/tensor/dense/dense_tensor_view.h> #include <vespa/vespalib/util/overload.h> #include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/util/typify.h> @@ -44,20 +45,19 @@ struct ConcatParam } }; -template <typename LCT, typename RCT> +template <typename LCT, typename RCT, typename OCT> std::unique_ptr<Value> generic_concat(const Value &a, const Value &b, const SparseJoinPlan &sparse_plan, const DenseConcatPlan &dense_plan, const ValueType &res_type, const ValueBuilderFactory &factory) { - using OCT = typename eval::UnifyCellTypes<LCT, RCT>::type; auto a_cells = a.cells().typify<LCT>(); auto b_cells = b.cells().typify<RCT>(); SparseJoinState sparse(sparse_plan, a.index(), b.index()); auto builder = factory.create_value_builder<OCT>(res_type, sparse_plan.sources.size(), - dense_plan.right.output_size, + dense_plan.output_size, sparse.first_index.size()); auto outer = sparse.first_index.create_view({}); auto inner = sparse.second_index.create_view(sparse.second_view_dims); @@ -81,29 +81,59 @@ generic_concat(const Value &a, const Value &b, return builder->build(std::move(builder)); } -template <typename LCT, typename RCT> +template <typename LCT, typename RCT, typename OCT> void my_generic_concat_op(State &state, uint64_t param_in) { const auto ¶m = unwrap_param<ConcatParam>(param_in); const Value &lhs = state.peek(1); const Value &rhs = state.peek(0); - auto res_value = generic_concat<LCT, RCT>(lhs, rhs, param.sparse_plan, param.dense_plan, - param.res_type, param.factory); + auto res_value = generic_concat<LCT, RCT, OCT>( + lhs, rhs, + param.sparse_plan, param.dense_plan, + param.res_type, param.factory); auto &result = state.stash.create<std::unique_ptr<Value>>(std::move(res_value)); const Value &result_ref = *(result.get()); state.pop_pop_push(result_ref); } +template <typename LCT, typename RCT, typename OCT> +void my_dense_simple_concat_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<ConcatParam>(param_in); + const Value &lhs = state.peek(1); + const Value &rhs = state.peek(0); + const auto a = lhs.cells().typify<LCT>(); + const auto b = rhs.cells().typify<RCT>(); + ArrayRef<OCT> result = state.stash.create_array<OCT>(a.size() + b.size()); + auto pos = result.begin(); + for (size_t i = 0; i < a.size(); ++i) { + *pos++ = a[i]; + } + for (size_t i = 0; i < b.size(); ++i) { + *pos++ = b[i]; + } + Value &ref = state.stash.create<tensor::DenseTensorView>(param.res_type, TypedCells(result)); + state.pop_pop_push(ref); +} + struct SelectGenericConcatOp { - template <typename LCT, typename RCT> static auto invoke() { - return my_generic_concat_op<LCT, RCT>; + template <typename LCT, typename RCT, typename OCT> static auto invoke(const ConcatParam ¶m) { + if (param.sparse_plan.sources.empty() && param.res_type.is_dense()) { + auto & dp = param.dense_plan; + if ((dp.output_size == (dp.left.input_size + dp.right.input_size)) + && (dp.right_offset == dp.left.input_size)) + { + return my_dense_simple_concat_op<LCT, RCT, OCT>; + } + } + return my_generic_concat_op<LCT, RCT, OCT>; } }; -enum class Case { NONE, OUT, BOTH }; +enum class Case { NONE, OUT, CONCAT, BOTH }; } // namespace <unnamed> -DenseConcatPlan::InOutLoop::InOutLoop(const ValueType &in_type, +std::pair<size_t, size_t> +DenseConcatPlan::InOutLoop::fill_from(const ValueType &in_type, std::string concat_dimension, const ValueType &out_type) { @@ -126,19 +156,21 @@ DenseConcatPlan::InOutLoop::InOutLoop(const ValueType &in_type, { [&](visit_ranges_first, const auto &) { abort(); }, [&](visit_ranges_second, const auto &b) { - if (b.name == concat_dimension) { update_plan(Case::OUT, 1, b.size, 0, 1); + if (b.name == concat_dimension) { update_plan(Case::CONCAT, 1, b.size, 0, 1); } else { update_plan(Case::OUT, b.size, b.size, 0, 1); } }, - [&](visit_ranges_both, const auto &a, const auto &b) { update_plan(Case::BOTH, a.size, b.size, 1, 1); } + [&](visit_ranges_both, const auto &a, const auto &b) { + if (b.name == concat_dimension) { update_plan(Case::CONCAT, a.size, b.size, 1, 1); + } else { update_plan(Case::BOTH, a.size, b.size, 1, 1); } + } }; - const auto input_dimensions = in_type.nontrivial_indexed_dimensions(); const auto output_dimensions = out_type.nontrivial_indexed_dimensions(); visit_ranges(visitor, input_dimensions.begin(), input_dimensions.end(), output_dimensions.begin(), output_dimensions.end(), [](const auto &a, const auto &b){ return (a.name < b.name); }); - input_size = 1; - output_size = 1; + size_t output_size = 1; + size_t offset_for_concat = 0; for (size_t i = in_loop_cnt.size(); i-- > 0; ) { if (in_stride[i] != 0) { in_stride[i] = input_size; @@ -148,7 +180,14 @@ DenseConcatPlan::InOutLoop::InOutLoop(const ValueType &in_type, assert(out_loop_cnt[i] != 0); out_stride[i] = output_size; output_size *= out_loop_cnt[i]; + // loop counts are different if and only if this is the concat dimension + if (in_loop_cnt[i] != out_loop_cnt[i]) { + assert(offset_for_concat == 0); + offset_for_concat = in_loop_cnt[i] * out_stride[i]; + } } + assert(offset_for_concat != 0); + return std::make_pair(offset_for_concat, output_size); } InterpretedFunction::Instruction @@ -157,8 +196,9 @@ GenericConcat::make_instruction(const ValueType &lhs_type, const ValueType &rhs_ const ValueBuilderFactory &factory, Stash &stash) { auto ¶m = stash.create<ConcatParam>(lhs_type, rhs_type, dimension, factory); - auto fun = typify_invoke<2,TypifyCellType,SelectGenericConcatOp>( - lhs_type.cell_type(), rhs_type.cell_type()); + auto fun = typify_invoke<3,TypifyCellType,SelectGenericConcatOp>( + lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), + param); return Instruction(fun, wrap_param<ConcatParam>(param)); } @@ -166,18 +206,11 @@ DenseConcatPlan::DenseConcatPlan(const ValueType &lhs_type, const ValueType &rhs_type, std::string concat_dimension, const ValueType &out_type) - : right_offset(0), - left(lhs_type, concat_dimension, out_type), - right(rhs_type, concat_dimension, out_type) { - const auto output_dimensions = out_type.nontrivial_indexed_dimensions(); - for (size_t i = 0; i < output_dimensions.size(); ++i) { - if (output_dimensions[i].name == concat_dimension) { - right_offset = left.in_loop_cnt[i] * left.out_stride[i]; - } - } - assert(right_offset > 0); - assert(left.output_size == right.output_size); + std::tie(right_offset, output_size) = left.fill_from(lhs_type, concat_dimension, out_type); + auto [ other_offset, other_size ] = right.fill_from(rhs_type, concat_dimension, out_type); + assert(other_offset > 0); + assert(output_size == other_size); } DenseConcatPlan::~DenseConcatPlan() = default; diff --git a/eval/src/vespa/eval/instruction/generic_concat.h b/eval/src/vespa/eval/instruction/generic_concat.h index c2636fb7678..5578c5a0dca 100644 --- a/eval/src/vespa/eval/instruction/generic_concat.h +++ b/eval/src/vespa/eval/instruction/generic_concat.h @@ -21,18 +21,19 @@ struct GenericConcat { struct DenseConcatPlan { size_t right_offset; + size_t output_size; struct InOutLoop { size_t input_size; - size_t output_size; std::vector<size_t> in_loop_cnt; std::vector<size_t> in_stride; std::vector<size_t> out_stride; + // returns computed concat offset and output size + std::pair<size_t, size_t> fill_from(const ValueType &in_type, + std::string concat_dimension, + const ValueType &out_type); template <typename F> void execute(size_t in_off, size_t out_off, const F &f) const { run_nested_loop(in_off, out_off, in_loop_cnt, in_stride, out_stride, f); } - InOutLoop(const ValueType &in_type, - std::string concat_dimension, - const ValueType &out_type); ~InOutLoop(); }; InOutLoop left; diff --git a/eval/src/vespa/eval/instruction/generic_join.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp index 53324924bdd..4e9dd8ded35 100644 --- a/eval/src/vespa/eval/instruction/generic_join.cpp +++ b/eval/src/vespa/eval/instruction/generic_join.cpp @@ -2,14 +2,17 @@ #include "generic_join.h" #include <vespa/eval/eval/inline_operation.h> +#include <vespa/eval/eval/fast_value.hpp> #include <vespa/vespalib/util/overload.h> #include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/util/visit_ranges.h> +#include <typeindex> #include <cassert> namespace vespalib::eval::instruction { +using operation::SwapArgs2; using State = InterpretedFunction::State; using Instruction = InterpretedFunction::Instruction; @@ -29,48 +32,70 @@ template <typename T> const T &unwrap_param(uint64_t param) { //----------------------------------------------------------------------------- -struct JoinParam { - ValueType res_type; - SparseJoinPlan sparse_plan; - DenseJoinPlan dense_plan; - join_fun_t function; - const ValueBuilderFactory &factory; - JoinParam(const ValueType &lhs_type, const ValueType &rhs_type, - join_fun_t function_in, const ValueBuilderFactory &factory_in) - : res_type(ValueType::join(lhs_type, rhs_type)), - sparse_plan(lhs_type, rhs_type), - dense_plan(lhs_type, rhs_type), - function(function_in), - factory(factory_in) - { - assert(!res_type.is_error()); +template <typename LCT, typename RCT, typename OCT, typename Fun> +void my_mixed_join_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<JoinParam>(param_in); + Fun fun(param.function); + auto dense_join = [&](const LCT *my_lhs, const RCT *my_rhs, OCT *my_res) + { + param.dense_plan.execute(0, 0, [&](size_t lhs_idx, size_t rhs_idx) { + *my_res++ = fun(my_lhs[lhs_idx], my_rhs[rhs_idx]); + }); + }; + const Value &lhs = state.peek(1); + const Value &rhs = state.peek(0); + auto lhs_cells = lhs.cells().typify<LCT>(); + auto rhs_cells = rhs.cells().typify<RCT>(); + SparseJoinState sparse(param.sparse_plan, lhs.index(), rhs.index()); + auto builder = param.factory.create_value_builder<OCT>(param.res_type, param.sparse_plan.sources.size(), param.dense_plan.out_size, sparse.first_index.size()); + auto outer = sparse.first_index.create_view({}); + auto inner = sparse.second_index.create_view(sparse.second_view_dims); + outer->lookup({}); + while (outer->next_result(sparse.first_address, sparse.first_subspace)) { + inner->lookup(sparse.address_overlap); + while (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { + dense_join(lhs_cells.begin() + param.dense_plan.lhs_size * sparse.lhs_subspace, + rhs_cells.begin() + param.dense_plan.rhs_size * sparse.rhs_subspace, + builder->add_subspace(sparse.full_address).begin()); + } } - ~JoinParam(); + auto &result = state.stash.create<std::unique_ptr<Value>>(builder->build(std::move(builder))); + const Value &result_ref = *(result.get()); + state.pop_pop_push(result_ref); }; -JoinParam::~JoinParam() = default; //----------------------------------------------------------------------------- - template <typename LCT, typename RCT, typename OCT, typename Fun> -void my_mixed_join_op(State &state, uint64_t param_in) { +void my_sparse_full_overlap_join_op(State &state, uint64_t param_in) { const auto ¶m = unwrap_param<JoinParam>(param_in); - Fun fun(param.function); const Value &lhs = state.peek(1); const Value &rhs = state.peek(0); auto lhs_cells = lhs.cells().typify<LCT>(); auto rhs_cells = rhs.cells().typify<RCT>(); - SparseJoinState sparse(param.sparse_plan, lhs.index(), rhs.index()); + const Value::Index &lhs_index = lhs.index(); + const Value::Index &rhs_index = rhs.index(); + if ((std::type_index(typeid(lhs_index)) == std::type_index(typeid(FastValueIndex))) && + (std::type_index(typeid(rhs_index)) == std::type_index(typeid(FastValueIndex)))) + { + const FastValueIndex &lhs_fast = static_cast<const FastValueIndex&>(lhs_index); + const FastValueIndex &rhs_fast = static_cast<const FastValueIndex&>(rhs_index); + return (rhs_fast.map.size() < lhs_fast.map.size()) + ? state.pop_pop_push(FastValueIndex::sparse_full_overlap_join<RCT,LCT,OCT,SwapArgs2<Fun>> + (param.res_type, SwapArgs2<Fun>(param.function), rhs_fast, lhs_fast, rhs_cells, lhs_cells, state.stash)) + : state.pop_pop_push(FastValueIndex::sparse_full_overlap_join<LCT,RCT,OCT,Fun> + (param.res_type, Fun(param.function), lhs_fast, rhs_fast, lhs_cells, rhs_cells, state.stash)); + } + Fun fun(param.function); + SparseJoinState sparse(param.sparse_plan, lhs_index, rhs_index); auto builder = param.factory.create_value_builder<OCT>(param.res_type, param.sparse_plan.sources.size(), param.dense_plan.out_size, sparse.first_index.size()); auto outer = sparse.first_index.create_view({}); auto inner = sparse.second_index.create_view(sparse.second_view_dims); outer->lookup({}); while (outer->next_result(sparse.first_address, sparse.first_subspace)) { inner->lookup(sparse.address_overlap); - while (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { - OCT *dst = builder->add_subspace(sparse.full_address).begin(); - auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; - param.dense_plan.execute(param.dense_plan.lhs_size * sparse.lhs_subspace, param.dense_plan.rhs_size * sparse.rhs_subspace, join_cells); + if (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { + builder->add_subspace(sparse.full_address)[0] = fun(lhs_cells[sparse.lhs_subspace], rhs_cells[sparse.rhs_subspace]); } } auto &result = state.stash.create<std::unique_ptr<Value>>(builder->build(std::move(builder))); @@ -78,6 +103,8 @@ void my_mixed_join_op(State &state, uint64_t param_in) { state.pop_pop_push(result_ref); }; +//----------------------------------------------------------------------------- + template <typename LCT, typename RCT, typename OCT, typename Fun> void my_dense_join_op(State &state, uint64_t param_in) { const auto ¶m = unwrap_param<JoinParam>(param_in); @@ -91,6 +118,8 @@ void my_dense_join_op(State &state, uint64_t param_in) { state.pop_pop_push(state.stash.create<DenseValueView>(param.res_type, TypedCells(out_cells))); }; +//----------------------------------------------------------------------------- + template <typename Fun> void my_double_join_op(State &state, uint64_t param_in) { Fun fun(unwrap_param<JoinParam>(param_in).function); @@ -98,6 +127,8 @@ void my_double_join_op(State &state, uint64_t param_in) { state.peek(0).cells().typify<double>()[0]))); }; +//----------------------------------------------------------------------------- + struct SelectGenericJoinOp { template <typename LCT, typename RCT, typename OCT, typename Fun> static auto invoke(const JoinParam ¶m) { if (param.res_type.is_double()) { @@ -110,6 +141,11 @@ struct SelectGenericJoinOp { if (param.sparse_plan.sources.empty()) { return my_dense_join_op<LCT,RCT,OCT,Fun>; } + if ((param.dense_plan.out_size == 1) && + (param.sparse_plan.sources.size() == param.sparse_plan.lhs_overlap.size())) + { + return my_sparse_full_overlap_join_op<LCT,RCT,OCT,Fun>; + } return my_mixed_join_op<LCT,RCT,OCT,Fun>; } }; @@ -221,6 +257,10 @@ SparseJoinState::~SparseJoinState() = default; //----------------------------------------------------------------------------- +JoinParam::~JoinParam() = default; + +//----------------------------------------------------------------------------- + using JoinTypify = TypifyValue<TypifyCellType,operation::TypifyOp2>; Instruction diff --git a/eval/src/vespa/eval/instruction/generic_join.h b/eval/src/vespa/eval/instruction/generic_join.h index 30e78b52510..8e878fc759f 100644 --- a/eval/src/vespa/eval/instruction/generic_join.h +++ b/eval/src/vespa/eval/instruction/generic_join.h @@ -79,6 +79,28 @@ struct SparseJoinState { ~SparseJoinState(); }; +/** + * Full set of parameters passed to low-level generic join function + **/ +struct JoinParam { + ValueType res_type; + SparseJoinPlan sparse_plan; + DenseJoinPlan dense_plan; + join_fun_t function; + const ValueBuilderFactory &factory; + JoinParam(const ValueType &lhs_type, const ValueType &rhs_type, + join_fun_t function_in, const ValueBuilderFactory &factory_in) + : res_type(ValueType::join(lhs_type, rhs_type)), + sparse_plan(lhs_type, rhs_type), + dense_plan(lhs_type, rhs_type), + function(function_in), + factory(factory_in) + { + assert(!res_type.is_error()); + } + ~JoinParam(); +}; + //----------------------------------------------------------------------------- } // namespace diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index 79f6f7e2a4f..b75b34098f5 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(eval_tensor OBJECT SOURCES default_tensor_engine.cpp default_value_builder_factory.cpp + partial_update.cpp tensor.cpp tensor_address.cpp wrapped_simple_tensor.cpp diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index 7d4bff21380..11c1ce74eca 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -445,8 +445,7 @@ struct CallAppendVector { template <typename OCT> void append_vector(OCT *&pos, const Value &value) { if (auto tensor = value.as_tensor()) { - const DenseTensorView *view = static_cast<const DenseTensorView *>(tensor); - dispatch_1<CallAppendVector<OCT> >(view->cellsRef(), pos); + dispatch_1<CallAppendVector<OCT> >(tensor->cells(), pos); } else { *pos++ = value.as_double(); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp index 84da53c8488..988116f53a6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp @@ -18,7 +18,7 @@ namespace { template <typename CT> void my_cell_range_op(eval::InterpretedFunction::State &state, uint64_t param) { const auto *self = (const DenseCellRangeFunction *)(param); - auto old_cells = DenseTensorView::typify_cells<CT>(state.peek(0)); + auto old_cells = state.peek(0).cells().typify<CT>(); ConstArrayRef<CT> new_cells(&old_cells[self->offset()], self->length()); state.pop_push(state.stash.create<DenseTensorView>(self->result_type(), TypedCells(new_cells))); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index 9e30451cd67..09530beb0b1 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -21,8 +21,8 @@ namespace { template <typename LCT, typename RCT> void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { - auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); - auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); + auto lhs_cells = state.peek(1).cells().typify<LCT>(); + auto rhs_cells = state.peek(0).cells().typify<RCT>(); double result = 0.0; const LCT *lhs = lhs_cells.cbegin(); const RCT *rhs = rhs_cells.cbegin(); @@ -33,15 +33,15 @@ void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { } void my_cblas_double_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { - auto lhs_cells = DenseTensorView::typify_cells<double>(state.peek(1)); - auto rhs_cells = DenseTensorView::typify_cells<double>(state.peek(0)); + auto lhs_cells = state.peek(1).cells().typify<double>(); + auto rhs_cells = state.peek(0).cells().typify<double>(); double result = cblas_ddot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1); state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); } void my_cblas_float_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { - auto lhs_cells = DenseTensorView::typify_cells<float>(state.peek(1)); - auto rhs_cells = DenseTensorView::typify_cells<float>(state.peek(0)); + auto lhs_cells = state.peek(1).cells().typify<float>(); + auto rhs_cells = state.peek(0).cells().typify<float>(); double result = cblas_sdot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1); state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_generic_join.hpp b/eval/src/vespa/eval/tensor/dense/dense_generic_join.hpp index cdc89b30fff..033ea3631be 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_generic_join.hpp +++ b/eval/src/vespa/eval/tensor/dense/dense_generic_join.hpp @@ -53,14 +53,10 @@ template <typename Function> std::unique_ptr<Tensor> generic_join(const DenseTensorView &lhs, const Tensor &rhs, Function &&func) { - const DenseTensorView *view = dynamic_cast<const DenseTensorView *>(&rhs); - if (view) { - DenseDimensionCombiner combiner(lhs.fast_type(), view->fast_type()); - TypedCells lhsCells = lhs.cellsRef(); - TypedCells rhsCells = view->cellsRef(); - return dispatch_2<CallGenericJoin>(lhsCells, rhsCells, combiner, std::move(func)); - } - return Tensor::UP(); + DenseDimensionCombiner combiner(lhs.fast_type(), rhs.type()); + TypedCells lhsCells = lhs.cells(); + TypedCells rhsCells = rhs.cells(); + return dispatch_2<CallGenericJoin>(lhsCells, rhsCells, combiner, std::move(func)); } } diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp index 70bdc8ae7d6..95cc702f85f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp @@ -34,7 +34,7 @@ template <typename DST_CT, typename SRC_CT> void my_lambda_peek_op(InterpretedFunction::State &state, uint64_t param) { const auto *self = (const Self *)(param); const std::vector<uint32_t> &lookup_table = self->table_token->get(); - auto src_cells = DenseTensorView::typify_cells<SRC_CT>(state.peek(0)); + auto src_cells = state.peek(0).cells().typify<SRC_CT>(); ArrayRef<DST_CT> dst_cells = state.stash.create_array<DST_CT>(lookup_table.size()); DST_CT *dst = &dst_cells[0]; for (uint32_t idx: lookup_table) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp index 9c18cf285d4..2973e190f79 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp @@ -36,8 +36,8 @@ template <typename LCT, typename RCT, bool lhs_common_inner, bool rhs_common_inn void my_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) { const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param)); using OCT = typename eval::UnifyCellTypes<LCT,RCT>::type; - auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); - auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); + auto lhs_cells = state.peek(1).cells().typify<LCT>(); + auto rhs_cells = state.peek(0).cells().typify<RCT>(); auto dst_cells = state.stash.create_array<OCT>(self.lhs_size * self.rhs_size); OCT *dst = dst_cells.begin(); const LCT *lhs = lhs_cells.cbegin(); @@ -55,8 +55,8 @@ void my_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) { template <bool lhs_common_inner, bool rhs_common_inner> void my_cblas_double_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) { const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param)); - auto lhs_cells = DenseTensorView::typify_cells<double>(state.peek(1)); - auto rhs_cells = DenseTensorView::typify_cells<double>(state.peek(0)); + auto lhs_cells = state.peek(1).cells().typify<double>(); + auto rhs_cells = state.peek(0).cells().typify<double>(); auto dst_cells = state.stash.create_array<double>(self.lhs_size * self.rhs_size); cblas_dgemm(CblasRowMajor, lhs_common_inner ? CblasNoTrans : CblasTrans, rhs_common_inner ? CblasTrans : CblasNoTrans, self.lhs_size, self.rhs_size, self.common_size, 1.0, @@ -69,8 +69,8 @@ void my_cblas_double_matmul_op(eval::InterpretedFunction::State &state, uint64_t template <bool lhs_common_inner, bool rhs_common_inner> void my_cblas_float_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) { const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param)); - auto lhs_cells = DenseTensorView::typify_cells<float>(state.peek(1)); - auto rhs_cells = DenseTensorView::typify_cells<float>(state.peek(0)); + auto lhs_cells = state.peek(1).cells().typify<float>(); + auto rhs_cells = state.peek(0).cells().typify<float>(); auto dst_cells = state.stash.create_array<float>(self.lhs_size * self.rhs_size); cblas_sgemm(CblasRowMajor, lhs_common_inner ? CblasNoTrans : CblasTrans, rhs_common_inner ? CblasTrans : CblasNoTrans, self.lhs_size, self.rhs_size, self.common_size, 1.0, diff --git a/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp index b8832305640..5b4bcd9e23e 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp @@ -32,8 +32,8 @@ void my_cblas_double_multi_matmul_op(InterpretedFunction::State &state, uint64_t size_t rhs_block_size = self.rhs_size() * self.common_size(); size_t dst_block_size = self.lhs_size() * self.rhs_size(); size_t num_blocks = self.matmul_cnt(); - const CT *lhs = DenseTensorView::typify_cells<CT>(state.peek(1)).cbegin(); - const CT *rhs = DenseTensorView::typify_cells<CT>(state.peek(0)).cbegin(); + const CT *lhs = state.peek(1).cells().typify<CT>().cbegin(); + const CT *rhs = state.peek(0).cells().typify<CT>().cbegin(); auto dst_cells = state.stash.create_array<CT>(dst_block_size * num_blocks); CT *dst = dst_cells.begin(); for (size_t i = 0; i < num_blocks; ++i, lhs += lhs_block_size, rhs += rhs_block_size, dst += dst_block_size) { @@ -53,8 +53,8 @@ void my_cblas_float_multi_matmul_op(InterpretedFunction::State &state, uint64_t size_t rhs_block_size = self.rhs_size() * self.common_size(); size_t dst_block_size = self.lhs_size() * self.rhs_size(); size_t num_blocks = self.matmul_cnt(); - const CT *lhs = DenseTensorView::typify_cells<CT>(state.peek(1)).cbegin(); - const CT *rhs = DenseTensorView::typify_cells<CT>(state.peek(0)).cbegin(); + const CT *lhs = state.peek(1).cells().typify<CT>().cbegin(); + const CT *rhs = state.peek(0).cells().typify<CT>().cbegin(); auto dst_cells = state.stash.create_array<CT>(dst_block_size * num_blocks); CT *dst = dst_cells.begin(); for (size_t i = 0; i < num_blocks; ++i, lhs += lhs_block_size, rhs += rhs_block_size, dst += dst_block_size) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp index 925627c5684..c6636a6a583 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp @@ -44,7 +44,7 @@ void my_number_join_op(State &state, uint64_t param) { OP my_op((join_fun_t)param); const Value &tensor = state.peek(swap ? 0 : 1); CT number = state.peek(swap ? 1 : 0).as_double(); - auto src_cells = DenseTensorView::typify_cells<CT>(tensor); + auto src_cells = tensor.cells().typify<CT>(); auto dst_cells = make_dst_cells<CT, inplace>(src_cells, state.stash); apply_op2_vec_num(dst_cells.begin(), src_cells.begin(), number, dst_cells.size(), my_op); if (inplace) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp index e2dc2241555..6e4756dbe3c 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp @@ -15,14 +15,9 @@ using namespace eval::tensor_function; namespace { -TypedCells getCellsRef(const eval::Value &value) { - const DenseTensorView &denseTensor = static_cast<const DenseTensorView &>(value); - return denseTensor.cellsRef(); -} - void my_replace_type_op(eval::InterpretedFunction::State &state, uint64_t param) { const ValueType *type = (const ValueType *)(param); - TypedCells cells = getCellsRef(state.peek(0)); + TypedCells cells = state.peek(0).cells(); state.pop_push(state.stash.create<DenseTensorView>(*type, cells)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp index e57cfd25325..5105ebab4ff 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp @@ -48,8 +48,8 @@ void my_simple_expand_op(State &state, uint64_t param) { using OP = typename std::conditional<rhs_inner,SwapArgs2<Fun>,Fun>::type; const ExpandParams ¶ms = *(ExpandParams*)param; OP my_op(params.function); - auto inner_cells = DenseTensorView::typify_cells<ICT>(state.peek(rhs_inner ? 0 : 1)); - auto outer_cells = DenseTensorView::typify_cells<OCT>(state.peek(rhs_inner ? 1 : 0)); + auto inner_cells = state.peek(rhs_inner ? 0 : 1).cells().typify<ICT>(); + auto outer_cells = state.peek(rhs_inner ? 1 : 0).cells().typify<OCT>(); auto dst_cells = state.stash.create_array<DCT>(params.result_size); DCT *dst = dst_cells.begin(); for (OCT outer_cell: outer_cells) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp index 05de3d07c96..f6c072708cb 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp @@ -70,8 +70,8 @@ void my_simple_join_op(State &state, uint64_t param) { using OP = typename std::conditional<swap,SwapArgs2<Fun>,Fun>::type; const JoinParams ¶ms = *(JoinParams*)param; OP my_op(params.function); - auto pri_cells = DenseTensorView::typify_cells<PCT>(state.peek(swap ? 0 : 1)); - auto sec_cells = DenseTensorView::typify_cells<SCT>(state.peek(swap ? 1 : 0)); + auto pri_cells = state.peek(swap ? 0 : 1).cells().typify<PCT>(); + auto sec_cells = state.peek(swap ? 1 : 0).cells().typify<SCT>(); auto dst_cells = make_dst_cells<OCT, pri_mut>(pri_cells, state.stash); if (overlap == Overlap::FULL) { apply_op2_vec_vec(dst_cells.begin(), pri_cells.begin(), sec_cells.begin(), dst_cells.size(), my_op); diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp index b5f46fca70c..da9b681019e 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp @@ -40,7 +40,7 @@ template <typename CT, typename Fun, bool inplace> void my_simple_map_op(State &state, uint64_t param) { Fun my_fun((map_fun_t)param); auto const &child = state.peek(0); - auto src_cells = DenseTensorView::typify_cells<CT>(child); + auto src_cells = child.cells().typify<CT>(); auto dst_cells = make_dst_cells<CT, inplace>(src_cells, state.stash); apply_op1_vec(dst_cells.begin(), src_cells.begin(), dst_cells.size(), my_fun); if (!inplace) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp index 571bcb79c9f..0ae40c8dac6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp @@ -55,7 +55,7 @@ CT reduce_cells(const CT *src, size_t dim_size, size_t stride, AGGR &aggr) { template <typename CT, typename AGGR> void my_single_reduce_op(InterpretedFunction::State &state, uint64_t param) { const auto ¶ms = *(const Params *)(param); - const CT *src = DenseTensorView::typify_cells<CT>(state.peek(0)).cbegin(); + const CT *src = state.peek(0).cells().typify<CT>().cbegin(); auto dst_cells = state.stash.create_array<CT>(params.outer_size * params.inner_size); AGGR aggr; CT *dst = dst_cells.begin(); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp index 16c0b01b169..477062ba9a3 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp @@ -36,7 +36,7 @@ void my_tensor_peek_op(eval::InterpretedFunction::State &state, uint64_t param) } factor *= dim.second; } - auto cells = DenseTensorView::typify_cells<CT>(state.peek(0)); + auto cells = state.peek(0).cells().typify<CT>(); state.stack.pop_back(); const Value &result = state.stash.create<DoubleValue>(valid ? cells[idx] : 0.0); state.stack.emplace_back(result); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp index b845ccf93a5..9303488bbc5 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp @@ -39,48 +39,34 @@ dimensionsAsString(const eval::ValueType &type) return oss.str(); } -size_t -calcCellsSize(const eval::ValueType &type) -{ - size_t cellsSize = 1; - for (const auto &dim : type.dimensions()) { - cellsSize *= dim.size; - } - return cellsSize; -} - - void -checkCellsSize(const DenseTensorView &arg) +checkCellsSize(const eval::ValueType &type, TypedCells cells) { - auto cellsSize = calcCellsSize(arg.fast_type()); - if (arg.cellsRef().size != cellsSize) { + auto cellsSize = type.dense_subspace_size(); + if (cells.size != cellsSize) { throw IllegalStateException(make_string("wrong cell size, " "expected=%zu, " "actual=%zu", cellsSize, - arg.cellsRef().size)); + cells.size)); } } void -checkDimensions(const DenseTensorView &lhs, const DenseTensorView &rhs, +checkDimensions(const eval::ValueType &lhs, const eval::ValueType &rhs, vespalib::stringref operation) { - if (lhs.fast_type().dimensions() != rhs.fast_type().dimensions()) { + if (lhs.dimensions() != rhs.dimensions()) { throw IllegalStateException(make_string("mismatching dimensions for " "dense tensor %s, " "lhs dimensions = '%s', " "rhs dimensions = '%s'", operation.data(), - dimensionsAsString(lhs.fast_type()).c_str(), - dimensionsAsString(rhs.fast_type()).c_str())); + dimensionsAsString(lhs).c_str(), + dimensionsAsString(rhs).c_str())); } - checkCellsSize(lhs); - checkCellsSize(rhs); } - /* * Join the cells of two tensors. * @@ -124,26 +110,18 @@ struct CallJoin template <typename Function> Tensor::UP -joinDenseTensors(const DenseTensorView &lhs, const DenseTensorView &rhs, - Function &&func) -{ - TypedCells lhsCells = lhs.cellsRef(); - TypedCells rhsCells = rhs.cellsRef(); - return dispatch_2<CallJoin>(lhsCells, rhsCells, lhs.fast_type().dimensions(), std::move(func)); -} - -template <typename Function> -Tensor::UP joinDenseTensors(const DenseTensorView &lhs, const Tensor &rhs, vespalib::stringref operation, Function &&func) { - auto view = dynamic_cast<const DenseTensorView *>(&rhs); - if (view) { - checkDimensions(lhs, *view, operation); - return joinDenseTensors(lhs, *view, func); - } - return Tensor::UP(); + const auto & lhs_type = lhs.fast_type(); + const auto & rhs_type = rhs.type(); + TypedCells lhs_cells = lhs.cells(); + TypedCells rhs_cells = rhs.cells(); + checkDimensions(lhs_type, rhs_type, operation); + checkCellsSize(lhs_type, lhs_cells); + checkCellsSize(rhs_type, rhs_cells); + return dispatch_2<CallJoin>(lhs_cells, rhs_cells, lhs_type.dimensions(), std::move(func)); } bool sameCells(TypedCells lhs, TypedCells rhs) @@ -215,9 +193,8 @@ DenseTensorView::apply(const CellFunction &func) const bool DenseTensorView::equals(const Tensor &arg) const { - auto view = dynamic_cast<const DenseTensorView *>(&arg); - if (view) { - return *this == *view; + if (fast_type() == arg.type()) { + return sameCells(cells(), arg.cells()); } return false; } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h index 2cea3c855a6..27a032b784f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h @@ -51,12 +51,6 @@ public: return MemoryUsage(sz, sz, 0, 0); } - template <typename T> static ConstArrayRef<T> typify_cells(const eval::Value &self) { - return static_cast<const DenseTensorView &>(self).cellsRef().typify<T>(); - } - template <typename T> static ConstArrayRef<T> unsafe_typify_cells(const eval::Value &self) { - return static_cast<const DenseTensorView &>(self).cellsRef().unsafe_typify<T>(); - } protected: explicit DenseTensorView(const eval::ValueType &type_in) : _typeRef(type_in), diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index 968308d69c9..33c674fc4ab 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -36,8 +36,8 @@ template <typename LCT, typename RCT, bool common_inner> void my_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); using OCT = typename eval::UnifyCellTypes<LCT,RCT>::type; - auto vector_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); - auto matrix_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); + auto vector_cells = state.peek(1).cells().typify<LCT>(); + auto matrix_cells = state.peek(0).cells().typify<RCT>(); auto dst_cells = state.stash.create_array<OCT>(self.result_size); OCT *dst = dst_cells.begin(); const RCT *matrix = matrix_cells.cbegin(); @@ -51,8 +51,8 @@ void my_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { template <bool common_inner> void my_cblas_double_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); - auto vector_cells = DenseTensorView::typify_cells<double>(state.peek(1)); - auto matrix_cells = DenseTensorView::typify_cells<double>(state.peek(0)); + auto vector_cells = state.peek(1).cells().typify<double>(); + auto matrix_cells = state.peek(0).cells().typify<double>(); auto dst_cells = state.stash.create_array<double>(self.result_size); cblas_dgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans, common_inner ? self.result_size : self.vector_size, @@ -65,8 +65,8 @@ void my_cblas_double_xw_product_op(eval::InterpretedFunction::State &state, uint template <bool common_inner> void my_cblas_float_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); - auto vector_cells = DenseTensorView::typify_cells<float>(state.peek(1)); - auto matrix_cells = DenseTensorView::typify_cells<float>(state.peek(0)); + auto vector_cells = state.peek(1).cells().typify<float>(); + auto matrix_cells = state.peek(0).cells().typify<float>(); auto dst_cells = state.stash.create_array<float>(self.result_size); cblas_sgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans, common_inner ? self.result_size : self.vector_size, diff --git a/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp index b581ce787e3..5db533a4655 100644 --- a/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp +++ b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp @@ -346,7 +346,7 @@ template <typename T> void Onnx::EvalContext::adapt_param(EvalContext &self, size_t idx, const eval::Value ¶m) { - const auto &cells_ref = static_cast<const DenseTensorView &>(param).cellsRef(); + const auto &cells_ref = param.cells(); auto cells = unconstify(cells_ref.typify<T>()); const auto &sizes = self._wire_info.onnx_inputs[idx].dimensions; self._param_values[idx] = Ort::Value::CreateTensor<T>(self._cpu_memory, cells.begin(), cells.size(), sizes.data(), sizes.size()); @@ -356,7 +356,7 @@ template <typename SRC, typename DST> void Onnx::EvalContext::convert_param(EvalContext &self, size_t idx, const eval::Value ¶m) { - auto cells = static_cast<const DenseTensorView &>(param).cellsRef().typify<SRC>(); + auto cells = param.cells().typify<SRC>(); size_t n = cells.size(); const SRC *src = cells.begin(); DST *dst = self._param_values[idx].GetTensorMutableData<DST>(); @@ -369,7 +369,7 @@ template <typename SRC, typename DST> void Onnx::EvalContext::convert_result(EvalContext &self, size_t idx) { - const auto &cells_ref = static_cast<const DenseTensorView &>(*self._results[idx]).cellsRef(); + const auto &cells_ref = (*self._results[idx]).cells(); auto cells = unconstify(cells_ref.typify<DST>()); size_t n = cells.size(); DST *dst = cells.begin(); diff --git a/eval/src/vespa/eval/tensor/partial_update.cpp b/eval/src/vespa/eval/tensor/partial_update.cpp new file mode 100644 index 00000000000..60d3dd638d1 --- /dev/null +++ b/eval/src/vespa/eval/tensor/partial_update.cpp @@ -0,0 +1,358 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "partial_update.h" +#include <vespa/vespalib/util/overload.h> +#include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/util/visit_ranges.h> +#include <cassert> +#include <set> + +#include <vespa/log/log.h> +LOG_SETUP(".eval.tensor.partial_update"); + +using namespace vespalib::eval; + +namespace vespalib::tensor { + +namespace { + +using join_fun_t = double (*)(double, double); + +static constexpr size_t npos() { return -1; } + +enum class DimCase { + MAPPED_MATCH, CONV_TO_INDEXED +}; + +struct DenseCoords { + std::vector<size_t> dim_sizes; + size_t total_size; + size_t offset; + size_t current; + DenseCoords(const ValueType &output_type) + : total_size(1), offset(0), current(0) + { + for (const auto & dim : output_type.dimensions()) { + if (dim.is_indexed()) { + dim_sizes.push_back(dim.size); + total_size *= dim.size; + } + } + } + ~DenseCoords(); + void clear() { offset = 0; current = 0; } + void convert_label(vespalib::stringref label) { + uint32_t coord = 0; + for (char c : label) { + if (c < '0' || c > '9') { // bad char + offset = npos(); + break; + } + coord = coord * 10 + (c - '0'); + } + size_t cur_dim_size = dim_sizes[current]; + if (coord < cur_dim_size) { + if (offset != npos()) { + offset *= cur_dim_size; + offset += coord; + } + } else { + offset = npos(); + } + ++current; + } + size_t get_dense_index() const { + assert(current == dim_sizes.size()); + return offset; + } +}; +DenseCoords::~DenseCoords() = default; + +struct SparseCoords { + std::vector<vespalib::stringref> addr; + std::vector<vespalib::stringref *> next_result_refs; + std::vector<const vespalib::stringref *> lookup_refs; + std::vector<size_t> lookup_view_dims; + SparseCoords(size_t sz) + : addr(sz), next_result_refs(sz), lookup_refs(sz), lookup_view_dims(sz) + { + for (size_t i = 0; i < sz; ++i) { + next_result_refs[i] = &addr[i]; + lookup_refs[i] = &addr[i]; + lookup_view_dims[i] = i; + } + } + ~SparseCoords(); +}; +SparseCoords::~SparseCoords() = default; + +/** + * Helper class that converts a fully-sparse address from the modifier + * tensor into a subset sparse address for the output and an offset + * in the dense subspace. + **/ +struct AddressHandler { + std::vector<DimCase> dimension_plan; + DenseCoords dense_converter; + SparseCoords for_output; + SparseCoords from_modifier; + bool valid; + + AddressHandler(const ValueType &output_type, + const ValueType &modifier_type) + : dimension_plan(), dense_converter(output_type), + for_output(output_type.count_mapped_dimensions()), + from_modifier(modifier_type.count_mapped_dimensions()), + valid(true) + { + if (! modifier_type.is_sparse()) { + LOG(error, "Unexpected non-sparse modifier tensor, type is %s", + modifier_type.to_spec().c_str()); + valid = false; + return; + } + // analyse dimensions + auto visitor = overload { + [&](visit_ranges_either, const auto &) { valid = false; }, + [&](visit_ranges_both, const auto &a, const auto &) { + dimension_plan.push_back(a.is_mapped() ? DimCase::MAPPED_MATCH : DimCase::CONV_TO_INDEXED); + } + }; + const auto & input_dims = output_type.dimensions(); + const auto & modifier_dims = modifier_type.dimensions(); + visit_ranges(visitor, + input_dims.begin(), input_dims.end(), + modifier_dims.begin(), modifier_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); + if (! valid) { + LOG(error, "Value type %s does not match modifier type %s (should have same dimensions)", + output_type.to_spec().c_str(), modifier_type.to_spec().c_str()); + return; + } + // implicitly checked above, must hold: + assert(input_dims.size() == modifier_dims.size()); + // the plan should now be fully built: + assert(input_dims.size() == dimension_plan.size()); + } + + void handle_address() + { + dense_converter.clear(); + auto out = for_output.addr.begin(); + for (size_t i = 0; i < dimension_plan.size(); ++i) { + if (dimension_plan[i] == DimCase::CONV_TO_INDEXED) { + dense_converter.convert_label(from_modifier.addr[i]); + } else { + *out++ = from_modifier.addr[i]; + } + } + assert(out == for_output.addr.end()); + assert(dense_converter.current == dense_converter.dim_sizes.size()); + } + + ~AddressHandler(); +}; +AddressHandler::~AddressHandler() = default; + +template <typename CT, typename ICT = CT, typename KeepFun> +void copy_tensor_with_filter(const Value &input, + size_t dsss, + SparseCoords &addrs, + ValueBuilder<CT> &builder, + KeepFun && keep_subspace) +{ + const auto input_cells = input.cells().typify<ICT>(); + auto input_view = input.index().create_view({}); + input_view->lookup({}); + size_t input_subspace_index; + while (input_view->next_result(addrs.next_result_refs, input_subspace_index)) { + if (keep_subspace(addrs.lookup_refs, input_subspace_index)) { + size_t input_offset = dsss * input_subspace_index; + auto src = input_cells.begin() + input_offset; + auto dst = builder.add_subspace(addrs.addr).begin(); + for (size_t i = 0; i < dsss; ++i) { + dst[i] = src[i]; + } + } + } +} + +template <typename CT> +Value::UP +copy_tensor(const Value &input, const ValueType &input_type, SparseCoords &helper, const ValueBuilderFactory &factory) +{ + const size_t num_mapped_in_input = input_type.count_mapped_dimensions(); + const size_t dsss = input_type.dense_subspace_size(); + const size_t expected_subspaces = input.index().size(); + auto builder = factory.create_value_builder<CT>(input_type, num_mapped_in_input, dsss, expected_subspaces); + auto no_filter = [] (const auto &, size_t) { + return true; + }; + copy_tensor_with_filter<CT>(input, dsss, helper, *builder, no_filter); + return builder->build(std::move(builder)); +} + +//----------------------------------------------------------------------------- + +struct PerformModify { + template<typename ICT, typename MCT> + static Value::UP invoke(const Value &input, + join_fun_t function, + const Value &modifier, + const ValueBuilderFactory &factory); +}; + +template <typename ICT, typename MCT> +Value::UP +PerformModify::invoke(const Value &input, join_fun_t function, const Value &modifier, const ValueBuilderFactory &factory) +{ + const ValueType &input_type = input.type(); + const size_t dsss = input_type.dense_subspace_size(); + const ValueType &modifier_type = modifier.type(); + AddressHandler handler(input_type, modifier_type); + if (! handler.valid) { + return Value::UP(); + } + // copy input to output + auto out = copy_tensor<ICT>(input, input_type, handler.for_output, factory); + // need to overwrite some cells + auto output_cells = unconstify(out->cells().template typify<ICT>()); + const auto modifier_cells = modifier.cells().typify<MCT>(); + auto modifier_view = modifier.index().create_view({}); + auto lookup_view = out->index().create_view(handler.for_output.lookup_view_dims); + modifier_view->lookup({}); + size_t modifier_subspace_index; + while (modifier_view->next_result(handler.from_modifier.next_result_refs, modifier_subspace_index)) { + handler.handle_address(); + size_t dense_idx = handler.dense_converter.get_dense_index(); + if (dense_idx == npos()) { + continue; + } + lookup_view->lookup(handler.for_output.lookup_refs); + size_t output_subspace_index; + if (lookup_view->next_result({}, output_subspace_index)) { + size_t subspace_offset = dsss * output_subspace_index; + auto dst = output_cells.begin() + subspace_offset; + ICT lhs = dst[dense_idx]; + MCT rhs = modifier_cells[modifier_subspace_index]; + dst[dense_idx] = function(lhs, rhs); + } + } + return out; +} + +//----------------------------------------------------------------------------- + +struct PerformAdd { + template<typename ICT, typename MCT> + static Value::UP invoke(const Value &input, + const Value &modifier, + const ValueBuilderFactory &factory); +}; + +template <typename ICT, typename MCT> +Value::UP +PerformAdd::invoke(const Value &input, const Value &modifier, const ValueBuilderFactory &factory) +{ + const ValueType &input_type = input.type(); + const ValueType &modifier_type = modifier.type(); + if (input_type.dimensions() != modifier_type.dimensions()) { + LOG(error, "when adding cells to a tensor, dimensions must be equal. " + "Got input type %s != modifier type %s", + input_type.to_spec().c_str(), modifier_type.to_spec().c_str()); + return Value::UP(); + } + const size_t num_mapped_in_input = input_type.count_mapped_dimensions(); + const size_t dsss = input_type.dense_subspace_size(); + const size_t expected_subspaces = input.index().size() + modifier.index().size(); + auto builder = factory.create_value_builder<ICT>(input_type, num_mapped_in_input, dsss, expected_subspaces); + SparseCoords addrs(num_mapped_in_input); + auto lookup_view = input.index().create_view(addrs.lookup_view_dims); + std::vector<bool> overwritten(input.index().size(), false); + auto remember_subspaces = [&] (const auto & lookup_refs, size_t) { + lookup_view->lookup(lookup_refs); + size_t input_subspace_index; + if (lookup_view->next_result({}, input_subspace_index)) { + overwritten[input_subspace_index] = true; + } + return true; + }; + copy_tensor_with_filter<ICT, MCT>(modifier, dsss, addrs, *builder, remember_subspaces); + auto filter = [&] (const auto &, size_t input_subspace) { + return ! overwritten[input_subspace]; + }; + copy_tensor_with_filter<ICT>(input, dsss, addrs, *builder, filter); + return builder->build(std::move(builder)); +} + +//----------------------------------------------------------------------------- + +struct PerformRemove { + template<typename ICT> + static Value::UP invoke(const Value &input, + const Value &modifier, + const ValueBuilderFactory &factory); +}; + +template <typename ICT> +Value::UP +PerformRemove::invoke(const Value &input, const Value &modifier, const ValueBuilderFactory &factory) +{ + const ValueType &input_type = input.type(); + const ValueType &modifier_type = modifier.type(); + if (input_type.mapped_dimensions() != modifier_type.dimensions()) { + LOG(error, "when removing cells from a tensor, mapped dimensions must be equal. " + "Got input type %s versus modifier type %s", + input_type.to_spec().c_str(), modifier_type.to_spec().c_str()); + return Value::UP(); + } + const size_t num_mapped_in_input = input_type.count_mapped_dimensions(); + if (num_mapped_in_input == 0) { + LOG(error, "cannot remove cells from a dense tensor of type %s", + input_type.to_spec().c_str()); + return Value::UP(); + } + SparseCoords addrs(num_mapped_in_input); + auto modifier_view = modifier.index().create_view(addrs.lookup_view_dims); + const size_t expected_subspaces = input.index().size(); + const size_t dsss = input_type.dense_subspace_size(); + auto builder = factory.create_value_builder<ICT>(input_type, num_mapped_in_input, dsss, expected_subspaces); + auto filter_by_modifier = [&] (const auto & lookup_refs, size_t) { + modifier_view->lookup(lookup_refs); + size_t modifier_subspace_index; + return !(modifier_view->next_result({}, modifier_subspace_index)); + }; + copy_tensor_with_filter<ICT>(input, dsss, addrs, *builder, filter_by_modifier); + return builder->build(std::move(builder)); +} + +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + +Value::UP +TensorPartialUpdate::modify(const Value &input, join_fun_t function, + const Value &modifier, const ValueBuilderFactory &factory) +{ + return typify_invoke<2, TypifyCellType, PerformModify>( + input.cells().type, modifier.cells().type, + input, function, modifier, factory); +} + +Value::UP +TensorPartialUpdate::add(const Value &input, const Value &add_cells, const ValueBuilderFactory &factory) +{ + return typify_invoke<2, TypifyCellType, PerformAdd>( + input.cells().type, add_cells.cells().type, + input, add_cells, factory); +} + +Value::UP +TensorPartialUpdate::remove(const Value &input, const Value &remove_spec, const ValueBuilderFactory &factory) +{ + return typify_invoke<1, TypifyCellType, PerformRemove>( + input.cells().type, + input, remove_spec, factory); +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/partial_update.h b/eval/src/vespa/eval/tensor/partial_update.h new file mode 100644 index 00000000000..8f997bde56c --- /dev/null +++ b/eval/src/vespa/eval/tensor/partial_update.h @@ -0,0 +1,42 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value.h> + +namespace vespalib::tensor { + +struct TensorPartialUpdate { + using join_fun_t = double (*)(double, double); + using Value = vespalib::eval::Value; + using ValueBuilderFactory = vespalib::eval::ValueBuilderFactory; + + /** + * Make a copy of the input, but apply function(oldvalue, modifier.cellvalue) + * to cells which also exist in the "modifier". + * The modifier type must be sparse with exactly the same dimension names + * as the input type. + * Returns null pointer if this constraint is violated. + **/ + static Value::UP modify(const Value &input, join_fun_t function, + const Value &modifier, const ValueBuilderFactory &factory); + + /** + * Make a copy of the input, but add or overwrite cells from add_cells. + * Requires same type for input and add_cells. + * Returns null pointer if this constraint is violated. + **/ + static Value::UP add(const Value &input, const Value &add_cells, const ValueBuilderFactory &factory); + + /** + * Make a copy of the input, but remove cells present in remove_spec. + * The remove_spec must be a sparse tensor, with exactly the mapped dimensions + * that the input value has. + * Cell values in remove_spec are ignored. + * Not valid for dense tensors, since removing cells for those are impossible. + * Returns null pointer if these constraints are violated. + **/ + static Value::UP remove(const Value &input, const Value &remove_spec, const ValueBuilderFactory &factory); +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h index 45fb7236152..e8838b1acdb 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h @@ -38,7 +38,7 @@ public: _hash(rhs._hash) {} - uint32_t hash() const { return _hash; } + uint32_t hash() const noexcept { return _hash; } uint32_t calcHash() const noexcept; diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp index 241b8026b59..178935fce00 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp @@ -145,7 +145,6 @@ WrappedSimpleTensor::add(const Tensor &arg) const if (!rhs || type() != rhs->type()) { return Tensor::UP(); } - TensorSpec oldTensor = toSpec(); TensorSpec argTensor = rhs->toSpec(); TensorSpec result(type().to_spec()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a49daf94f01..8d27c723b89 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -325,13 +325,6 @@ public class Flags { APPLICATION_ID ); - public static final UnboundBooleanFlag WEIGHTED_DNS_PER_REGION = defineFeatureFlag( - "weighted-dns-per-region", true, - "Whether to create weighted DNS records per region in global endpoints", - "Takes effect on next deployment through controller", - APPLICATION_ID - ); - public static final UnboundBooleanFlag ONLY_PUBLIC_ACCESS = defineFeatureFlag( "enable-public-only", false, "Only access public hosts from container", @@ -391,6 +384,12 @@ public class Flags { "Whether application containers should use the new restapi handler implementation", "Takes effect on next internal redeployment"); + public static final UnboundBooleanFlag ALWAYS_ACQUIRE_PROVISION_LOCK = defineFeatureFlag( + "always-acquire-provision-lock", + false, + "Whether provision lock should always be taken when writing nodes", + "Takes effect on config server restart"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index 62000efb682..3fe7b5b7614 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -4,7 +4,6 @@ #include <vespa/fnet/fnet.h> #include <vespa/vespalib/net/server_socket.h> #include <vespa/vespalib/net/crypto_engine.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/stringfmt.h> using namespace vespalib; diff --git a/fnet/src/tests/time/timespeed.cpp b/fnet/src/tests/time/timespeed.cpp index 1204b0ff334..e6d2af5a278 100644 --- a/fnet/src/tests/time/timespeed.cpp +++ b/fnet/src/tests/time/timespeed.cpp @@ -9,14 +9,14 @@ using vespalib::BenchmarkTimer; TEST("steady clock speed") { using clock = std::chrono::steady_clock; clock::time_point t; - double min_time_us = BenchmarkTimer::benchmark([&t](){t = clock::now();}, 1.0) * 1000000.0; + double min_time_us = BenchmarkTimer::benchmark([&t]() noexcept {t = clock::now();}, 1.0) * 1000000.0; fprintf(stderr, "approx overhead per sample (steady clock): %f us\n", min_time_us); } TEST("system clock speed") { using clock = std::chrono::system_clock; clock::time_point t; - double min_time_us = BenchmarkTimer::benchmark([&t](){t = clock::now();}, 1.0) * 1000000.0; + double min_time_us = BenchmarkTimer::benchmark([&t]() noexcept {t = clock::now();}, 1.0) * 1000000.0; fprintf(stderr, "approx overhead per sample (system clock): %f us\n", min_time_us); } diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 158524d1214..a20de880f15 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -7,7 +7,6 @@ #include "connector.h" #include "connection.h" #include "transport.h" -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/server_socket.h> #include <csignal> diff --git a/fsa/src/vespa/fsa/fsa.h b/fsa/src/vespa/fsa/fsa.h index e4d3246d924..9c668f1f85f 100644 --- a/fsa/src/vespa/fsa/fsa.h +++ b/fsa/src/vespa/fsa/fsa.h @@ -304,7 +304,7 @@ public: * * @param f Reference to FSA. */ - State(const FSA& f) : _fsa(&f), _state(_fsa->start()) {} + State(const FSA& f) noexcept : _fsa(&f), _state(_fsa->start()) {} /** * @brief Constructor. @@ -314,7 +314,7 @@ public: * * @param f Pointer to FSA. */ - State(const FSA* f) : _fsa(f), _state(_fsa->start()) {} + State(const FSA* f) noexcept : _fsa(f), _state(_fsa->start()) {} /** * @brief Copy constructor. @@ -325,14 +325,14 @@ public: * * @param s Reference to state to be duplicated. */ - State(const State& s) : _fsa(s._fsa), _state(s._state) {} + State(const State& s) noexcept : _fsa(s._fsa), _state(s._state) {} /** * @brief Destructor. * * Destructor, does nothing special. */ - virtual ~State() {} + virtual ~State() = default; /** * @brief Check if the automaton has perfect hash built in. @@ -1043,7 +1043,7 @@ public: * * @param f Reference to FSA. */ - WordCounterState(const FSA& f) : State(f), _counter(0) {} + WordCounterState(const FSA& f) noexcept : State(f), _counter(0) {} /** * @brief Constructor. @@ -1053,7 +1053,7 @@ public: * * @param f Pointer to FSA. */ - WordCounterState(const FSA* f) : State(f), _counter(0) {} + WordCounterState(const FSA* f) noexcept : State(f), _counter(0) {} /** * @brief Copy constructor. @@ -1062,12 +1062,12 @@ public: * * @param s Reference to hashed state to copy. */ - WordCounterState(const WordCounterState& s) : State(s), _counter(s._counter) {} + WordCounterState(const WordCounterState& s) noexcept : State(s), _counter(s._counter) {} /** * @brief Destructor. */ - virtual ~WordCounterState() {} + virtual ~WordCounterState() = default; /** * @brief Set the state to the starting state of the automaton. @@ -1798,7 +1798,7 @@ public: * * @param f Reference to FSA. */ - HashedWordCounterState(const FSA& f) : State(f), _hash(0), _counter(0) {} + HashedWordCounterState(const FSA& f) noexcept : State(f), _hash(0), _counter(0) {} /** * @brief Constructor. @@ -1808,7 +1808,7 @@ public: * * @param f Pointer to FSA. */ - HashedWordCounterState(const FSA* f) : State(f), _hash(0), _counter(0) {} + HashedWordCounterState(const FSA* f) noexcept : State(f), _hash(0), _counter(0) {} /** * @brief Copy constructor. @@ -1817,12 +1817,12 @@ public: * * @param s Reference to hashed state to copy. */ - HashedWordCounterState(const HashedWordCounterState& s) : State(s), _hash(s._hash), _counter(s._counter) {} + HashedWordCounterState(const HashedWordCounterState& s) noexcept : State(s), _hash(s._hash), _counter(s._counter) {} /** * @brief Destructor. */ - virtual ~HashedWordCounterState() {} + virtual ~HashedWordCounterState() = default; /** * @brief Set the state to the starting state of the automaton. @@ -2120,7 +2120,7 @@ public: * * @return Index of the start state (0 if the %FSA is empty). */ - state_t start() const + state_t start() const noexcept { return _start; } diff --git a/fsa/src/vespa/fsa/segmenter.h b/fsa/src/vespa/fsa/segmenter.h index a2720c6de0c..f2826f9c967 100644 --- a/fsa/src/vespa/fsa/segmenter.h +++ b/fsa/src/vespa/fsa/segmenter.h @@ -147,7 +147,7 @@ public: * * Null segment at postion zero. */ - Segment() : _beg(0), _end(0), _conn(0) {} + Segment() noexcept : _beg(0), _end(0), _conn(0) {} /** * @brief Constructor. @@ -156,7 +156,7 @@ public: * @param e End of the segment (the position after the last term). * @param c Connexity of the segment. */ - Segment(unsigned int b, unsigned int e, unsigned int c) : + Segment(unsigned int b, unsigned int e, unsigned int c) noexcept : _beg(b), _end(e), _conn(c) {} /** @@ -164,12 +164,12 @@ public: * * @param s Segment object to copy. */ - Segment(const Segment &s) : _beg(s._beg), _end(s._end), _conn(s._conn) {} + Segment(const Segment &s) noexcept : _beg(s._beg), _end(s._end), _conn(s._conn) {} /** * @brief Destructor. */ - ~Segment() {} + ~Segment() = default; /** * @brief Set the segment parameters. diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index f6bfe769997..3f68009cd42 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -682,6 +682,44 @@ ], "fields": [] }, + "com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.jdisc.http.ServerConfig$AccessLog)", + "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder remoteAddressHeaders(java.lang.String)", + "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder remoteAddressHeaders(java.util.Collection)", + "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder remotePortHeaders(java.lang.String)", + "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder remotePortHeaders(java.util.Collection)", + "public com.yahoo.jdisc.http.ServerConfig$AccessLog build()" + ], + "fields": [ + "public java.util.List remoteAddressHeaders", + "public java.util.List remotePortHeaders" + ] + }, + "com.yahoo.jdisc.http.ServerConfig$AccessLog": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder)", + "public java.util.List remoteAddressHeaders()", + "public java.lang.String remoteAddressHeaders(int)", + "public java.util.List remotePortHeaders()", + "public java.lang.String remotePortHeaders(int)" + ], + "fields": [] + }, "com.yahoo.jdisc.http.ServerConfig$Builder": { "superClass": "java.lang.Object", "interfaces": [ @@ -704,6 +742,7 @@ "public com.yahoo.jdisc.http.ServerConfig$Builder stopTimeout(double)", "public com.yahoo.jdisc.http.ServerConfig$Builder jmx(com.yahoo.jdisc.http.ServerConfig$Jmx$Builder)", "public com.yahoo.jdisc.http.ServerConfig$Builder metric(com.yahoo.jdisc.http.ServerConfig$Metric$Builder)", + "public com.yahoo.jdisc.http.ServerConfig$Builder accessLog(com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -713,7 +752,8 @@ "fields": [ "public java.util.List filter", "public com.yahoo.jdisc.http.ServerConfig$Jmx$Builder jmx", - "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder metric" + "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder metric", + "public com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder accessLog" ] }, "com.yahoo.jdisc.http.ServerConfig$Filter$Builder": { @@ -854,7 +894,8 @@ "public int maxWorkerThreads()", "public double stopTimeout()", "public com.yahoo.jdisc.http.ServerConfig$Jmx jmx()", - "public com.yahoo.jdisc.http.ServerConfig$Metric metric()" + "public com.yahoo.jdisc.http.ServerConfig$Metric metric()", + "public com.yahoo.jdisc.http.ServerConfig$AccessLog accessLog()" ], "fields": [ "public static final java.lang.String CONFIG_DEF_MD5", diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 7085f07585a..e8fd92f8e19 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -4,6 +4,7 @@ package com.yahoo.jdisc.http.server.jetty; import com.google.common.base.Objects; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.jdisc.http.servlet.ServletRequest; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.RequestLog; @@ -15,6 +16,7 @@ import java.security.Principal; import java.security.cert.X509Certificate; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; import java.util.logging.Level; import java.util.logging.Logger; @@ -27,25 +29,21 @@ import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnectorLoca * @author Oyvind Bakksjo * @author bjorncs */ -public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog { +class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog { private static final Logger logger = Logger.getLogger(AccessLogRequestLog.class.getName()); - // TODO These hardcoded headers should be provided by config instead - private static final String HEADER_NAME_X_FORWARDED_FOR = "x-forwarded-for"; - private static final String HEADER_NAME_X_FORWARDED_PORT = "X-Forwarded-Port"; - private static final String HEADER_NAME_Y_RA = "y-ra"; - private static final String HEADER_NAME_Y_RP = "y-rp"; - private static final String HEADER_NAME_YAHOOREMOTEIP = "yahooremoteip"; - private static final String HEADER_NAME_CLIENT_IP = "client-ip"; - // HTTP headers that are logged as extra key-value-pairs in access log entries private static final List<String> LOGGED_REQUEST_HEADERS = List.of("Vespa-Client-Version"); private final AccessLog accessLog; + private final List<String> remoteAddressHeaders; + private final List<String> remotePortHeaders; - public AccessLogRequestLog(AccessLog accessLog) { + AccessLogRequestLog(AccessLog accessLog, ServerConfig.AccessLog config) { this.accessLog = accessLog; + this.remoteAddressHeaders = config.remoteAddressHeaders(); + this.remotePortHeaders = config.remotePortHeaders(); } @Override @@ -121,26 +119,30 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog } } - private static String getRemoteAddress(HttpServletRequest request) { - return Optional.ofNullable(request.getHeader(HEADER_NAME_X_FORWARDED_FOR)) - .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_Y_RA))) - .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_YAHOOREMOTEIP))) - .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_CLIENT_IP))) - .orElseGet(request::getRemoteAddr); + private String getRemoteAddress(HttpServletRequest request) { + for (String header : remoteAddressHeaders) { + String value = request.getHeader(header); + if (value != null) return value; + } + return request.getRemoteAddr(); } - private static int getRemotePort(HttpServletRequest request) { - return Optional.ofNullable(request.getHeader(HEADER_NAME_X_FORWARDED_PORT)) - .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_Y_RP))) - .flatMap(AccessLogRequestLog::parsePort) - .orElseGet(request::getRemotePort); + private int getRemotePort(HttpServletRequest request) { + for (String header : remotePortHeaders) { + String value = request.getHeader(header); + if (value != null) { + OptionalInt maybePort = parsePort(value); + if (maybePort.isPresent()) return maybePort.getAsInt(); + } + } + return request.getRemotePort(); } - private static Optional<Integer> parsePort(String port) { + private static OptionalInt parsePort(String port) { try { - return Optional.of(Integer.valueOf(port)); + return OptionalInt.of(Integer.parseInt(port)); } catch (IllegalArgumentException e) { - return Optional.empty(); + return OptionalInt.empty(); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index cc7ed7ac3e0..2cab06e9e23 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -54,7 +54,7 @@ class HttpRequestDispatch { private final ServletResponseController servletResponseController; private final RequestHandler requestHandler; - private final MetricReporter metricReporter; + private final RequestMetricReporter metricReporter; public HttpRequestDispatch(JDiscContext jDiscContext, AccessLogEntry accessLogEntry, @@ -66,7 +66,7 @@ class HttpRequestDispatch { requestHandler = newRequestHandler(jDiscContext, accessLogEntry, servletRequest); this.jettyRequest = (Request) servletRequest; - this.metricReporter = new MetricReporter(jDiscContext.metric, metricContext, jettyRequest.getTimeStamp()); + this.metricReporter = new RequestMetricReporter(jDiscContext.metric, metricContext, jettyRequest.getTimeStamp()); this.servletResponseController = new ServletResponseController(servletRequest, servletResponse, jDiscContext.janitor, diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java index 31a8303ab4b..82c445c7ca9 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java @@ -2,7 +2,6 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer.Metrics; import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.server.AsyncContextEvent; import org.eclipse.jetty.server.Handler; @@ -51,13 +50,13 @@ public class HttpResponseStatisticsCollector extends HandlerWrapper implements G } private static final String[] HTTP_RESPONSE_GROUPS = { - Metrics.RESPONSES_1XX, - Metrics.RESPONSES_2XX, - Metrics.RESPONSES_3XX, - Metrics.RESPONSES_4XX, - Metrics.RESPONSES_5XX, - Metrics.RESPONSES_401, - Metrics.RESPONSES_403 + MetricDefinitions.RESPONSES_1XX, + MetricDefinitions.RESPONSES_2XX, + MetricDefinitions.RESPONSES_3XX, + MetricDefinitions.RESPONSES_4XX, + MetricDefinitions.RESPONSES_5XX, + MetricDefinitions.RESPONSES_401, + MetricDefinitions.RESPONSES_403 }; private final AtomicLong inFlight = new AtomicLong(); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java index dfbcfb741f5..8ffc6759ae7 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java @@ -87,8 +87,8 @@ class JDiscHttpServlet extends HttpServlet { request.setAttribute(JDiscServerConnector.REQUEST_ATTRIBUTE, getConnector(request)); Metric.Context metricContext = getMetricContext(request); - context.metric.add(JettyHttpServer.Metrics.NUM_REQUESTS, 1, metricContext); - context.metric.add(JettyHttpServer.Metrics.JDISC_HTTP_REQUESTS, 1, metricContext); + context.metric.add(MetricDefinitions.NUM_REQUESTS, 1, metricContext); + context.metric.add(MetricDefinitions.JDISC_HTTP_REQUESTS, 1, metricContext); String method = request.getMethod().toUpperCase(); if (servletSupportedMethods.contains(method)) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java index 824a8cda330..f82b51804a9 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java @@ -75,8 +75,8 @@ class JDiscServerConnector extends ServerConnector { var requestDimensions = new RequestDimensions(method, scheme); return requestMetricContextCache.computeIfAbsent(requestDimensions, ignored -> { Map<String, Object> dimensions = createConnectorDimensions(listenPort, connectorName); - dimensions.put(JettyHttpServer.Metrics.METHOD_DIMENSION, method); - dimensions.put(JettyHttpServer.Metrics.SCHEME_DIMENSION, scheme); + dimensions.put(MetricDefinitions.METHOD_DIMENSION, method); + dimensions.put(MetricDefinitions.SCHEME_DIMENSION, scheme); return metric.createContext(dimensions); }); } @@ -95,8 +95,8 @@ class JDiscServerConnector extends ServerConnector { private static Map<String, Object> createConnectorDimensions(int listenPort, String connectorName) { Map<String, Object> props = new HashMap<>(); - props.put(JettyHttpServer.Metrics.NAME_DIMENSION, connectorName); - props.put(JettyHttpServer.Metrics.PORT_DIMENSION, listenPort); + props.put(MetricDefinitions.NAME_DIMENSION, connectorName); + props.put(MetricDefinitions.PORT_DIMENSION, listenPort); return props; } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index c826f52a865..303b36430a5 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -1,11 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import com.google.common.annotations.Beta; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; @@ -15,7 +14,6 @@ import com.yahoo.jdisc.http.server.FilterBindings; import com.yahoo.jdisc.service.AbstractServerProvider; import com.yahoo.jdisc.service.CurrentContainer; import org.eclipse.jetty.http.HttpField; -import org.eclipse.jetty.io.ConnectionStatistics; import org.eclipse.jetty.jmx.ConnectorServer; import org.eclipse.jetty.jmx.MBeanContainer; import org.eclipse.jetty.server.Connector; @@ -23,7 +21,6 @@ import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; -import org.eclipse.jetty.server.handler.AbstractHandlerContainer; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.StatisticsHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; @@ -44,14 +41,9 @@ import java.net.MalformedURLException; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -62,77 +54,21 @@ import static java.util.stream.Collectors.toList; * @author Simon Thoresen Hult * @author bjorncs */ -@Beta public class JettyHttpServer extends AbstractServerProvider { - public interface Metrics { - String NAME_DIMENSION = "serverName"; - String PORT_DIMENSION = "serverPort"; - String METHOD_DIMENSION = "httpMethod"; - String SCHEME_DIMENSION = "scheme"; - String REQUEST_TYPE_DIMENSION = "requestType"; - String CLIENT_IP_DIMENSION = "clientIp"; - - String NUM_OPEN_CONNECTIONS = "serverNumOpenConnections"; - String NUM_CONNECTIONS_OPEN_MAX = "serverConnectionsOpenMax"; - String CONNECTION_DURATION_MAX = "serverConnectionDurationMax"; - String CONNECTION_DURATION_MEAN = "serverConnectionDurationMean"; - String CONNECTION_DURATION_STD_DEV = "serverConnectionDurationStdDev"; - String NUM_PREMATURELY_CLOSED_CONNECTIONS = "jdisc.http.request.prematurely_closed"; - - String NUM_BYTES_RECEIVED = "serverBytesReceived"; - String NUM_BYTES_SENT = "serverBytesSent"; - - String NUM_CONNECTIONS = "serverNumConnections"; - - /* For historical reasons, these are all aliases for the same metric. 'jdisc.http' should ideally be the only one. */ - String JDISC_HTTP_REQUESTS = "jdisc.http.requests"; - String NUM_REQUESTS = "serverNumRequests"; - - String NUM_SUCCESSFUL_RESPONSES = "serverNumSuccessfulResponses"; - String NUM_FAILED_RESPONSES = "serverNumFailedResponses"; - String NUM_SUCCESSFUL_WRITES = "serverNumSuccessfulResponseWrites"; - String NUM_FAILED_WRITES = "serverNumFailedResponseWrites"; - - String TOTAL_SUCCESSFUL_LATENCY = "serverTotalSuccessfulResponseLatency"; - String TOTAL_FAILED_LATENCY = "serverTotalFailedResponseLatency"; - String TIME_TO_FIRST_BYTE = "serverTimeToFirstByte"; - - String RESPONSES_1XX = "http.status.1xx"; - String RESPONSES_2XX = "http.status.2xx"; - String RESPONSES_3XX = "http.status.3xx"; - String RESPONSES_4XX = "http.status.4xx"; - String RESPONSES_5XX = "http.status.5xx"; - String RESPONSES_401 = "http.status.401"; - String RESPONSES_403 = "http.status.403"; - - String STARTED_MILLIS = "serverStartedMillis"; - - String URI_LENGTH = "jdisc.http.request.uri_length"; - String CONTENT_SIZE = "jdisc.http.request.content_size"; - - String SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.missing_client_cert"; - String SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.expired_client_cert"; - String SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.invalid_client_cert"; - String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS = "jdisc.http.ssl.handshake.failure.incompatible_protocols"; - String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS = "jdisc.http.ssl.handshake.failure.incompatible_ciphers"; - String SSL_HANDSHAKE_FAILURE_UNKNOWN = "jdisc.http.ssl.handshake.failure.unknown"; - } - private final static Logger log = Logger.getLogger(JettyHttpServer.class.getName()); - private final long timeStarted = System.currentTimeMillis(); + private final ExecutorService janitor; - private final ScheduledExecutorService metricReporterExecutor; - private final Metric metric; + private final Server server; private final List<Integer> listenedPorts = new ArrayList<>(); + private final ServerMetricReporter metricsReporter; @Inject public JettyHttpServer(CurrentContainer container, Metric metric, ServerConfig serverConfig, ServletPathsConfig servletPathsConfig, - ThreadFactory threadFactory, FilterBindings filterBindings, ComponentRegistry<ConnectorFactory> connectorFactories, ComponentRegistry<ServletHolder> servletHolders, @@ -141,13 +77,12 @@ public class JettyHttpServer extends AbstractServerProvider { super(container); if (connectorFactories.allComponents().isEmpty()) throw new IllegalArgumentException("No connectors configured."); - this.metric = metric; initializeJettyLogging(); server = new Server(); server.setStopTimeout((long)(serverConfig.stopTimeout() * 1000.0)); - server.setRequestLog(new AccessLogRequestLog(accessLog)); + server.setRequestLog(new AccessLogRequestLog(accessLog, serverConfig.accessLog())); setupJmx(server, serverConfig); ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(serverConfig.maxWorkerThreads()); @@ -157,7 +92,7 @@ public class JettyHttpServer extends AbstractServerProvider { listenedPorts.add(connectorConfig.listenPort()); } - janitor = newJanitor(threadFactory); + janitor = newJanitor(); JDiscContext jDiscContext = new JDiscContext(filterBindings.getRequestFilters().activate(), filterBindings.getResponseFilters().activate(), @@ -179,16 +114,7 @@ public class JettyHttpServer extends AbstractServerProvider { jdiscServlet, servletHolders, jDiscFilterInvokerFilter)); - - int numMetricReporterThreads = 1; - metricReporterExecutor = - Executors.newScheduledThreadPool(numMetricReporterThreads, - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat(JettyHttpServer.class.getName() + "-MetricReporter-%d") - .setThreadFactory(threadFactory) - .build()); - metricReporterExecutor.scheduleAtFixedRate(new MetricTask(), 0, 2, TimeUnit.SECONDS); + this.metricsReporter = new ServerMetricReporter(metric, server); } private static void initializeJettyLogging() { @@ -274,23 +200,19 @@ public class JettyHttpServer extends AbstractServerProvider { return ports.stream().map(Object::toString).collect(Collectors.joining(":")); } - private static ExecutorService newJanitor(ThreadFactory factory) { + private static ExecutorService newJanitor() { int threadPoolSize = Runtime.getRuntime().availableProcessors(); log.info("Creating janitor executor with " + threadPoolSize + " threads"); return Executors.newFixedThreadPool( threadPoolSize, - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat(JettyHttpServer.class.getName() + "-Janitor-%d") - .setThreadFactory(factory) - .build() - ); + new DaemonThreadFactory(JettyHttpServer.class.getName() + "-Janitor-")); } @Override public void start() { try { server.start(); + metricsReporter.start(); logEffectiveSslConfiguration(); } catch (final Exception e) { if (e instanceof IOException && e.getCause() instanceof BindException) { @@ -326,7 +248,7 @@ public class JettyHttpServer extends AbstractServerProvider { log.log(Level.SEVERE, "Server shutdown threw an unexpected exception.", e); } - metricReporterExecutor.shutdown(); + metricsReporter.shutdown(); janitor.shutdown(); } @@ -340,56 +262,6 @@ public class JettyHttpServer extends AbstractServerProvider { Server server() { return server; } - private class MetricTask implements Runnable { - @Override - public void run() { - HttpResponseStatisticsCollector statisticsCollector = ((AbstractHandlerContainer) server.getHandler()) - .getChildHandlerByClass(HttpResponseStatisticsCollector.class); - if (statisticsCollector != null) { - setServerMetrics(statisticsCollector); - } - - // reset statisticsHandler to preserve earlier behavior - StatisticsHandler statisticsHandler = ((AbstractHandlerContainer) server.getHandler()) - .getChildHandlerByClass(StatisticsHandler.class); - if (statisticsHandler != null) { - statisticsHandler.statsReset(); - } - - for (Connector connector : server.getConnectors()) { - setConnectorMetrics((JDiscServerConnector)connector); - } - } - - } - - private void setServerMetrics(HttpResponseStatisticsCollector statisticsCollector) { - long timeSinceStarted = System.currentTimeMillis() - timeStarted; - metric.set(Metrics.STARTED_MILLIS, timeSinceStarted, null); - - addResponseMetrics(statisticsCollector); - } - - private void addResponseMetrics(HttpResponseStatisticsCollector statisticsCollector) { - for (var metricEntry : statisticsCollector.takeStatistics()) { - Map<String, Object> dimensions = new HashMap<>(); - dimensions.put(Metrics.METHOD_DIMENSION, metricEntry.method); - dimensions.put(Metrics.SCHEME_DIMENSION, metricEntry.scheme); - dimensions.put(Metrics.REQUEST_TYPE_DIMENSION, metricEntry.requestType); - metric.add(metricEntry.name, metricEntry.value, metric.createContext(dimensions)); - } - } - - private void setConnectorMetrics(JDiscServerConnector connector) { - ConnectionStatistics statistics = connector.getStatistics(); - metric.set(Metrics.NUM_CONNECTIONS, statistics.getConnectionsTotal(), connector.getConnectorMetricContext()); - metric.set(Metrics.NUM_OPEN_CONNECTIONS, statistics.getConnections(), connector.getConnectorMetricContext()); - metric.set(Metrics.NUM_CONNECTIONS_OPEN_MAX, statistics.getConnectionsMax(), connector.getConnectorMetricContext()); - metric.set(Metrics.CONNECTION_DURATION_MAX, statistics.getConnectionDurationMax(), connector.getConnectorMetricContext()); - metric.set(Metrics.CONNECTION_DURATION_MEAN, statistics.getConnectionDurationMean(), connector.getConnectorMetricContext()); - metric.set(Metrics.CONNECTION_DURATION_STD_DEV, statistics.getConnectionDurationStdDev(), connector.getConnectorMetricContext()); - } - private StatisticsHandler newStatisticsHandler() { StatisticsHandler statisticsHandler = new StatisticsHandler(); statisticsHandler.statsReset(); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java new file mode 100644 index 00000000000..a55a6e8a734 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -0,0 +1,71 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +/** + * Name and dimensions for jdisc/container metrics + * + * @author bjorncs + */ +class MetricDefinitions { + static final String NAME_DIMENSION = "serverName"; + static final String PORT_DIMENSION = "serverPort"; + static final String METHOD_DIMENSION = "httpMethod"; + static final String SCHEME_DIMENSION = "scheme"; + static final String REQUEST_TYPE_DIMENSION = "requestType"; + static final String CLIENT_IP_DIMENSION = "clientIp"; + + static final String NUM_OPEN_CONNECTIONS = "serverNumOpenConnections"; + static final String NUM_CONNECTIONS_OPEN_MAX = "serverConnectionsOpenMax"; + static final String CONNECTION_DURATION_MAX = "serverConnectionDurationMax"; + static final String CONNECTION_DURATION_MEAN = "serverConnectionDurationMean"; + static final String CONNECTION_DURATION_STD_DEV = "serverConnectionDurationStdDev"; + static final String NUM_PREMATURELY_CLOSED_CONNECTIONS = "jdisc.http.request.prematurely_closed"; + + static final String NUM_BYTES_RECEIVED = "serverBytesReceived"; + static final String NUM_BYTES_SENT = "serverBytesSent"; + + static final String NUM_CONNECTIONS = "serverNumConnections"; + + /* For historical reasons, these are all aliases for the same metric. 'jdisc.http' should ideally be the only one. */ + static final String JDISC_HTTP_REQUESTS = "jdisc.http.requests"; + static final String NUM_REQUESTS = "serverNumRequests"; + + static final String NUM_SUCCESSFUL_RESPONSES = "serverNumSuccessfulResponses"; + static final String NUM_FAILED_RESPONSES = "serverNumFailedResponses"; + static final String NUM_SUCCESSFUL_WRITES = "serverNumSuccessfulResponseWrites"; + static final String NUM_FAILED_WRITES = "serverNumFailedResponseWrites"; + + static final String TOTAL_SUCCESSFUL_LATENCY = "serverTotalSuccessfulResponseLatency"; + static final String TOTAL_FAILED_LATENCY = "serverTotalFailedResponseLatency"; + static final String TIME_TO_FIRST_BYTE = "serverTimeToFirstByte"; + + static final String RESPONSES_1XX = "http.status.1xx"; + static final String RESPONSES_2XX = "http.status.2xx"; + static final String RESPONSES_3XX = "http.status.3xx"; + static final String RESPONSES_4XX = "http.status.4xx"; + static final String RESPONSES_5XX = "http.status.5xx"; + static final String RESPONSES_401 = "http.status.401"; + static final String RESPONSES_403 = "http.status.403"; + + static final String STARTED_MILLIS = "serverStartedMillis"; + + static final String URI_LENGTH = "jdisc.http.request.uri_length"; + static final String CONTENT_SIZE = "jdisc.http.request.content_size"; + + static final String SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.missing_client_cert"; + static final String SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.expired_client_cert"; + static final String SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.invalid_client_cert"; + static final String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS = "jdisc.http.ssl.handshake.failure.incompatible_protocols"; + static final String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS = "jdisc.http.ssl.handshake.failure.incompatible_ciphers"; + static final String SSL_HANDSHAKE_FAILURE_UNKNOWN = "jdisc.http.ssl.handshake.failure.unknown"; + + static final String JETTY_THREADPOOL_MAX_THREADS = "jdisc.http.jetty.threadpool.thread.max"; + static final String JETTY_THREADPOOL_MIN_THREADS = "jdisc.http.jetty.threadpool.thread.min"; + static final String JETTY_THREADPOOL_RESERVED_THREADS = "jdisc.http.jetty.threadpool.thread.reserved"; + static final String JETTY_THREADPOOL_BUSY_THREADS = "jdisc.http.jetty.threadpool.thread.busy"; + static final String JETTY_THREADPOOL_IDLE_THREADS = "jdisc.http.jetty.threadpool.thread.idle"; + static final String JETTY_THREADPOOL_TOTAL_THREADS = "jdisc.http.jetty.threadpool.thread.total"; + static final String JETTY_THREADPOOL_QUEUE_SIZE = "jdisc.http.jetty.threadpool.queue.size"; + + private MetricDefinitions() {} +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/RequestMetricReporter.java index 21a64792731..7596be0415a 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/RequestMetricReporter.java @@ -4,8 +4,6 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Metric.Context; -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer.Metrics; - import java.util.concurrent.atomic.AtomicBoolean; @@ -13,7 +11,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * Responsible for metric reporting for JDisc http request handler support. * @author Tony Vaagenes */ -public class MetricReporter { +class RequestMetricReporter { private final Metric metric; private final Context context; @@ -23,65 +21,65 @@ public class MetricReporter { private final AtomicBoolean firstSetOfTimeToFirstByte = new AtomicBoolean(true); - public MetricReporter(Metric metric, Context context, long requestStartTime) { + RequestMetricReporter(Metric metric, Context context, long requestStartTime) { this.metric = metric; this.context = context; this.requestStartTime = requestStartTime; } - public void successfulWrite(int numBytes) { + void successfulWrite(int numBytes) { setTimeToFirstByteFirstTime(); - metric.add(Metrics.NUM_SUCCESSFUL_WRITES, 1, context); - metric.set(Metrics.NUM_BYTES_SENT, numBytes, context); + metric.add(MetricDefinitions.NUM_SUCCESSFUL_WRITES, 1, context); + metric.set(MetricDefinitions.NUM_BYTES_SENT, numBytes, context); } private void setTimeToFirstByteFirstTime() { boolean isFirstWrite = firstSetOfTimeToFirstByte.getAndSet(false); if (isFirstWrite) { long timeToFirstByte = getRequestLatency(); - metric.set(Metrics.TIME_TO_FIRST_BYTE, timeToFirstByte, context); + metric.set(MetricDefinitions.TIME_TO_FIRST_BYTE, timeToFirstByte, context); } } - public void failedWrite() { - metric.add(Metrics.NUM_FAILED_WRITES, 1, context); + void failedWrite() { + metric.add(MetricDefinitions.NUM_FAILED_WRITES, 1, context); } - public void successfulResponse() { + void successfulResponse() { setTimeToFirstByteFirstTime(); long requestLatency = getRequestLatency(); - metric.set(Metrics.TOTAL_SUCCESSFUL_LATENCY, requestLatency, context); + metric.set(MetricDefinitions.TOTAL_SUCCESSFUL_LATENCY, requestLatency, context); - metric.add(Metrics.NUM_SUCCESSFUL_RESPONSES, 1, context); + metric.add(MetricDefinitions.NUM_SUCCESSFUL_RESPONSES, 1, context); } - public void failedResponse() { + void failedResponse() { setTimeToFirstByteFirstTime(); - metric.set(Metrics.TOTAL_FAILED_LATENCY, getRequestLatency(), context); - metric.add(Metrics.NUM_FAILED_RESPONSES, 1, context); + metric.set(MetricDefinitions.TOTAL_FAILED_LATENCY, getRequestLatency(), context); + metric.add(MetricDefinitions.NUM_FAILED_RESPONSES, 1, context); } - public void prematurelyClosed() { - metric.add(Metrics.NUM_PREMATURELY_CLOSED_CONNECTIONS, 1, context); + void prematurelyClosed() { + metric.add(MetricDefinitions.NUM_PREMATURELY_CLOSED_CONNECTIONS, 1, context); } - public void successfulRead(int bytes_received) { - metric.set(JettyHttpServer.Metrics.NUM_BYTES_RECEIVED, bytes_received, context); + void successfulRead(int bytes_received) { + metric.set(MetricDefinitions.NUM_BYTES_RECEIVED, bytes_received, context); } private long getRequestLatency() { return System.currentTimeMillis() - requestStartTime; } - public void uriLength(int length) { - metric.set(Metrics.URI_LENGTH, length, context); + void uriLength(int length) { + metric.set(MetricDefinitions.URI_LENGTH, length, context); } - public void contentSize(int size) { - metric.set(Metrics.CONTENT_SIZE, size, context); + void contentSize(int size) { + metric.set(MetricDefinitions.CONTENT_SIZE, size, context); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServerMetricReporter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServerMetricReporter.java new file mode 100644 index 00000000000..ba3694ffc2f --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServerMetricReporter.java @@ -0,0 +1,115 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.jdisc.Metric; +import org.eclipse.jetty.io.ConnectionStatistics; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.AbstractHandlerContainer; +import org.eclipse.jetty.server.handler.StatisticsHandler; +import org.eclipse.jetty.util.thread.QueuedThreadPool; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Reports server/connector specific metrics for Jdisc and Jetty + * + * @author bjorncs + */ +class ServerMetricReporter { + + private final ScheduledExecutorService executor = + Executors.newScheduledThreadPool(1, new DaemonThreadFactory("jdisc-jetty-metric-reporter-")); + private final Metric metric; + private final Server jetty; + + ServerMetricReporter(Metric metric, Server jetty) { + this.metric = metric; + this.jetty = jetty; + } + + void start() { + executor.scheduleAtFixedRate(new ReporterTask(), 0, 2, TimeUnit.SECONDS); + } + + void shutdown() { + try { + executor.shutdownNow(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private class ReporterTask implements Runnable { + + private final Instant timeStarted = Instant.now(); + + @Override + public void run() { + HttpResponseStatisticsCollector statisticsCollector = ((AbstractHandlerContainer) jetty.getHandler()) + .getChildHandlerByClass(HttpResponseStatisticsCollector.class); + if (statisticsCollector != null) { + setServerMetrics(statisticsCollector); + } + + // reset statisticsHandler to preserve earlier behavior + StatisticsHandler statisticsHandler = ((AbstractHandlerContainer) jetty.getHandler()) + .getChildHandlerByClass(StatisticsHandler.class); + if (statisticsHandler != null) { + statisticsHandler.statsReset(); + } + + for (Connector connector : jetty.getConnectors()) { + setConnectorMetrics((JDiscServerConnector)connector); + } + + setJettyThreadpoolMetrics(); + } + + private void setServerMetrics(HttpResponseStatisticsCollector statisticsCollector) { + long timeSinceStarted = System.currentTimeMillis() - timeStarted.toEpochMilli(); + metric.set(MetricDefinitions.STARTED_MILLIS, timeSinceStarted, null); + + addResponseMetrics(statisticsCollector); + } + + private void addResponseMetrics(HttpResponseStatisticsCollector statisticsCollector) { + for (var metricEntry : statisticsCollector.takeStatistics()) { + Map<String, Object> dimensions = new HashMap<>(); + dimensions.put(MetricDefinitions.METHOD_DIMENSION, metricEntry.method); + dimensions.put(MetricDefinitions.SCHEME_DIMENSION, metricEntry.scheme); + dimensions.put(MetricDefinitions.REQUEST_TYPE_DIMENSION, metricEntry.requestType); + metric.add(metricEntry.name, metricEntry.value, metric.createContext(dimensions)); + } + } + + private void setJettyThreadpoolMetrics() { + QueuedThreadPool threadpool = (QueuedThreadPool) jetty.getThreadPool(); + metric.set(MetricDefinitions.JETTY_THREADPOOL_MAX_THREADS, threadpool.getMaxThreads(), null); + metric.set(MetricDefinitions.JETTY_THREADPOOL_MIN_THREADS, threadpool.getMinThreads(), null); + metric.set(MetricDefinitions.JETTY_THREADPOOL_RESERVED_THREADS, threadpool.getReservedThreads(), null); + metric.set(MetricDefinitions.JETTY_THREADPOOL_BUSY_THREADS, threadpool.getBusyThreads(), null); + metric.set(MetricDefinitions.JETTY_THREADPOOL_IDLE_THREADS, threadpool.getIdleThreads(), null); + metric.set(MetricDefinitions.JETTY_THREADPOOL_TOTAL_THREADS, threadpool.getThreads(), null); + metric.set(MetricDefinitions.JETTY_THREADPOOL_QUEUE_SIZE, threadpool.getQueueSize(), null); + } + + private void setConnectorMetrics(JDiscServerConnector connector) { + ConnectionStatistics statistics = connector.getStatistics(); + metric.set(MetricDefinitions.NUM_CONNECTIONS, statistics.getConnectionsTotal(), connector.getConnectorMetricContext()); + metric.set(MetricDefinitions.NUM_OPEN_CONNECTIONS, statistics.getConnections(), connector.getConnectorMetricContext()); + metric.set(MetricDefinitions.NUM_CONNECTIONS_OPEN_MAX, statistics.getConnectionsMax(), connector.getConnectorMetricContext()); + metric.set(MetricDefinitions.CONNECTION_DURATION_MAX, statistics.getConnectionDurationMax(), connector.getConnectorMetricContext()); + metric.set(MetricDefinitions.CONNECTION_DURATION_MEAN, statistics.getConnectionDurationMean(), connector.getConnectorMetricContext()); + metric.set(MetricDefinitions.CONNECTION_DURATION_STD_DEV, statistics.getConnectionDurationStdDev(), connector.getConnectorMetricContext()); + } + + } +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java index a764c75f766..b4d03385c3b 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java @@ -59,7 +59,7 @@ public class ServletOutputStreamWriter { // GuardedBy("monitor") private final Deque<ResponseContentPart> responseContentQueue = new ArrayDeque<>(); - private final MetricReporter metricReporter; + private final RequestMetricReporter metricReporter; /** * When this future completes there will be no more calls against the servlet output stream or servlet response. @@ -70,7 +70,7 @@ public class ServletOutputStreamWriter { final CompletableFuture<Void> finishedFuture = new CompletableFuture<>(); - public ServletOutputStreamWriter(ServletOutputStream outputStream, Executor executor, MetricReporter metricReporter) { + public ServletOutputStreamWriter(ServletOutputStream outputStream, Executor executor, RequestMetricReporter metricReporter) { this.outputStream = outputStream; this.executor = executor; this.metricReporter = metricReporter; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index fd1f84f7d49..9e7912f2dc1 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -43,7 +43,7 @@ class ServletRequestReader implements ReadListener { private final ContentChannel requestContentChannel; private final Executor executor; - private final MetricReporter metricReporter; + private final RequestMetricReporter metricReporter; private int bytesRead; @@ -94,7 +94,7 @@ class ServletRequestReader implements ReadListener { ServletInputStream servletInputStream, ContentChannel requestContentChannel, Executor executor, - MetricReporter metricReporter) { + RequestMetricReporter metricReporter) { Preconditions.checkNotNull(servletInputStream); Preconditions.checkNotNull(requestContentChannel); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java index 5dd6b72dc20..60b7878156f 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java @@ -58,7 +58,7 @@ public class ServletResponseController { HttpServletRequest servletRequest, HttpServletResponse servletResponse, Executor executor, - MetricReporter metricReporter, + RequestMetricReporter metricReporter, boolean developerMode) throws IOException { this.servletRequest = servletRequest; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java index fc9a6fc03be..4c9059b5b37 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java @@ -2,7 +2,6 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer.Metrics; import org.eclipse.jetty.io.ssl.SslHandshakeListener; import javax.net.ssl.SSLHandshakeException; @@ -38,36 +37,36 @@ class SslHandshakeFailedListener implements SslHandshakeListener { log.log(Level.FINE, throwable, () -> "Ssl handshake failed: " + throwable.getMessage()); String metricName = SslHandshakeFailure.fromSslHandshakeException((SSLHandshakeException) throwable) .map(SslHandshakeFailure::metricName) - .orElse(Metrics.SSL_HANDSHAKE_FAILURE_UNKNOWN); + .orElse(MetricDefinitions.SSL_HANDSHAKE_FAILURE_UNKNOWN); metric.add(metricName, 1L, metric.createContext(createDimensions(event))); } private Map<String, Object> createDimensions(Event event) { Map<String, Object> dimensions = new HashMap<>(); - dimensions.put(Metrics.NAME_DIMENSION, connectorName); - dimensions.put(Metrics.PORT_DIMENSION, listenPort); + dimensions.put(MetricDefinitions.NAME_DIMENSION, connectorName); + dimensions.put(MetricDefinitions.PORT_DIMENSION, listenPort); Optional.ofNullable(event.getSSLEngine().getPeerHost()) - .ifPresent(clientIp -> dimensions.put(Metrics.CLIENT_IP_DIMENSION, clientIp)); + .ifPresent(clientIp -> dimensions.put(MetricDefinitions.CLIENT_IP_DIMENSION, clientIp)); return Map.copyOf(dimensions); } private enum SslHandshakeFailure { INCOMPATIBLE_PROTOCOLS( - Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, + MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, "(Client requested protocol \\S+? is not enabled or supported in server context" + "|The client supported protocol versions \\[\\S+?\\] are not accepted by server preferences \\[\\S+?\\])"), INCOMPATIBLE_CIPHERS( - Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, + MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, "no cipher suites in common"), MISSING_CLIENT_CERT( - Metrics.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, + MetricDefinitions.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, "Empty server certificate chain"), EXPIRED_CLIENT_CERTIFICATE( - Metrics.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, + MetricDefinitions.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, // Note: this pattern will match certificates with too late notBefore as well "PKIX path validation failed: java.security.cert.CertPathValidatorException: validity check failed"), INVALID_CLIENT_CERT( - Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, // Includes mismatch of client certificate and private key + MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, // Includes mismatch of client certificate and private key "(PKIX path (building|validation) failed: .+)|(Invalid CertificateVerify signature)"); private final String metricName; diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def index 0cb5b89b20c..3118a7dea64 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def @@ -43,4 +43,10 @@ jmx.listenPort int default = 1099 metric.monitoringHandlerPaths[] string # Paths that should be reported with search dimensions where applicable -metric.searchHandlerPaths[] string
\ No newline at end of file +metric.searchHandlerPaths[] string + +# HTTP request headers that contain remote address +accessLog.remoteAddressHeaders[] string + +# HTTP request headers that contain remote port +accessLog.remotePortHeaders[] string diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java index 69535be034c..a4fd7c9bc5f 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java @@ -3,6 +3,7 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.http.ServerConfig; import org.eclipse.jetty.http.MetaData; import org.eclipse.jetty.server.HttpChannel; import org.eclipse.jetty.server.HttpConnection; @@ -11,6 +12,7 @@ import org.eclipse.jetty.server.Response; import org.eclipse.jetty.server.ServerConnector; import org.junit.Test; +import java.util.List; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; @@ -33,7 +35,7 @@ public class AccessLogRequestLogTest { when(jettyRequest.getRequestURI()).thenReturn("/search/"); when(jettyRequest.getQueryString()).thenReturn("query=year:>2010"); - new AccessLogRequestLog(mock(AccessLog.class)).log(jettyRequest, createResponseMock()); + doAccessLoggingOfRequest(jettyRequest); assertThat(accessLogEntry.getRawPath(), is(not(nullValue()))); assertTrue(accessLogEntry.getRawQuery().isPresent()); @@ -48,7 +50,7 @@ public class AccessLogRequestLogTest { final String query = "query=year%252010+%3B&customParameter=something"; when(jettyRequest.getQueryString()).thenReturn(query); - new AccessLogRequestLog(mock(AccessLog.class)).log(jettyRequest, createResponseMock()); + doAccessLoggingOfRequest(jettyRequest); assertThat(accessLogEntry.getRawPath(), is(path)); assertThat(accessLogEntry.getRawQuery().get(), is(query)); @@ -64,7 +66,7 @@ public class AccessLogRequestLogTest { String rawQuery = "q=%%2"; when(jettyRequest.getQueryString()).thenReturn(rawQuery); - new AccessLogRequestLog(mock(AccessLog.class)).log(jettyRequest, createResponseMock()); + doAccessLoggingOfRequest(jettyRequest); assertThat(accessLogEntry.getRawPath(), is(rawPath)); Optional<String> actualRawQuery = accessLogEntry.getRawQuery(); assertThat(actualRawQuery.isPresent(), is(true)); @@ -80,7 +82,7 @@ public class AccessLogRequestLogTest { when(jettyRequest.getHeader("x-forwarded-for")).thenReturn("1.2.3.4"); when(jettyRequest.getHeader("y-ra")).thenReturn("2.3.4.5"); - new AccessLogRequestLog(mock(AccessLog.class)).log(jettyRequest, createResponseMock()); + doAccessLoggingOfRequest(jettyRequest); assertThat(accessLogEntry.getRemoteAddress(), is("1.2.3.4")); } @@ -93,7 +95,7 @@ public class AccessLogRequestLogTest { when(jettyRequest.getHeader("X-Forwarded-Port")).thenReturn("80"); when(jettyRequest.getHeader("y-rp")).thenReturn("8080"); - new AccessLogRequestLog(mock(AccessLog.class)).log(jettyRequest, createResponseMock()); + doAccessLoggingOfRequest(jettyRequest); assertThat(accessLogEntry.getRemotePort(), is(80)); } @@ -105,11 +107,19 @@ public class AccessLogRequestLogTest { when(jettyRequest.getHeader("X-Forwarded-Port")).thenReturn("8o8o"); when(jettyRequest.getRemotePort()).thenReturn(80); - new AccessLogRequestLog(mock(AccessLog.class)).log(jettyRequest, createResponseMock()); + doAccessLoggingOfRequest(jettyRequest); assertThat(accessLogEntry.getRemotePort(), is(0)); assertThat(accessLogEntry.getPeerPort(), is(80)); } + private void doAccessLoggingOfRequest(Request jettyRequest) { + ServerConfig.AccessLog config = new ServerConfig.AccessLog( + new ServerConfig.AccessLog.Builder() + .remoteAddressHeaders(List.of("x-forwarded-for", "y-ra")) + .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); + new AccessLogRequestLog(mock(AccessLog.class), config).log(jettyRequest, createResponseMock()); + } + private static Request createRequestMock(AccessLogEntry entry) { ServerConnector serverConnector = mock(ServerConnector.class); when(serverConnector.getLocalPort()).thenReturn(1234); diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java index 4ae824e2b7a..bb92d75bed5 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java @@ -2,7 +2,6 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.http.server.jetty.HttpResponseStatisticsCollector.StatisticsEntry; -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer.Metrics; import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpURI; import org.eclipse.jetty.http.HttpVersion; @@ -48,8 +47,8 @@ public class HttpResponseStatisticsCollectorTest { testRequest("http", 200, "GET"); var stats = collector.takeStatistics(); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_2XX, 1L); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_3XX, 2L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_2XX, 1L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_3XX, 2L); } @Test @@ -66,12 +65,12 @@ public class HttpResponseStatisticsCollectorTest { testRequest("https", 200, "POST"); var stats = collector.takeStatistics(); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_2XX, 1L); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_4XX, 1L); - assertStatisticsEntryPresent(stats, "http", "PUT", Metrics.RESPONSES_2XX, 1L); - assertStatisticsEntryPresent(stats, "http", "POST", Metrics.RESPONSES_2XX, 2L); - assertStatisticsEntryPresent(stats, "https", "GET", Metrics.RESPONSES_4XX, 1L); - assertStatisticsEntryPresent(stats, "https", "POST", Metrics.RESPONSES_2XX, 4L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_2XX, 1L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_4XX, 1L); + assertStatisticsEntryPresent(stats, "http", "PUT", MetricDefinitions.RESPONSES_2XX, 1L); + assertStatisticsEntryPresent(stats, "http", "POST", MetricDefinitions.RESPONSES_2XX, 2L); + assertStatisticsEntryPresent(stats, "https", "GET", MetricDefinitions.RESPONSES_4XX, 1L); + assertStatisticsEntryPresent(stats, "https", "POST", MetricDefinitions.RESPONSES_2XX, 4L); } @Test @@ -81,9 +80,9 @@ public class HttpResponseStatisticsCollectorTest { testRequest("http", 403, "GET"); var stats = collector.takeStatistics(); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_4XX, 3L); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_401, 1L); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_403, 1L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_4XX, 3L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_401, 1L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_403, 1L); } @@ -93,12 +92,12 @@ public class HttpResponseStatisticsCollectorTest { testRequest("http", 200, "GET"); var stats = collector.takeStatistics(); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_2XX, 2L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_2XX, 2L); testRequest("http", 200, "GET"); stats = collector.takeStatistics(); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_2XX, 1L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_2XX, 1L); } @Test @@ -109,15 +108,15 @@ public class HttpResponseStatisticsCollectorTest { testRequest("http", 200, "GET", "/status.html?foo=bar"); var stats = collector.takeStatistics(); - assertStatisticsEntryWithRequestTypePresent(stats, "http", "GET", Metrics.RESPONSES_2XX, "monitoring", 1L); - assertStatisticsEntryWithRequestTypePresent(stats, "http", "GET", Metrics.RESPONSES_2XX, "read", 1L); - assertStatisticsEntryWithRequestTypePresent(stats, "http", "POST", Metrics.RESPONSES_2XX, "read", 1L); - assertStatisticsEntryWithRequestTypePresent(stats, "http", "POST", Metrics.RESPONSES_2XX, "write", 1L); + assertStatisticsEntryWithRequestTypePresent(stats, "http", "GET", MetricDefinitions.RESPONSES_2XX, "monitoring", 1L); + assertStatisticsEntryWithRequestTypePresent(stats, "http", "GET", MetricDefinitions.RESPONSES_2XX, "read", 1L); + assertStatisticsEntryWithRequestTypePresent(stats, "http", "POST", MetricDefinitions.RESPONSES_2XX, "read", 1L); + assertStatisticsEntryWithRequestTypePresent(stats, "http", "POST", MetricDefinitions.RESPONSES_2XX, "write", 1L); testRequest("http", 200, "GET"); stats = collector.takeStatistics(); - assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_2XX, 1L); + assertStatisticsEntryPresent(stats, "http", "GET", MetricDefinitions.RESPONSES_2XX, 1L); } @Test @@ -125,7 +124,7 @@ public class HttpResponseStatisticsCollectorTest { testRequest("http", 200, "GET", "/search", com.yahoo.jdisc.Request.RequestType.WRITE); var stats = collector.takeStatistics(); - assertStatisticsEntryWithRequestTypePresent(stats, "http", "GET", Metrics.RESPONSES_2XX, "write", 1L); + assertStatisticsEntryWithRequestTypePresent(stats, "http", "GET", MetricDefinitions.RESPONSES_2XX, "write", 1L); } @Before diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 96cf1d4c01f..01ba776fbf0 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -23,7 +23,6 @@ import com.yahoo.jdisc.http.Cookie; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.HttpResponse; import com.yahoo.jdisc.http.ServerConfig; -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer.Metrics; import com.yahoo.jdisc.http.server.jetty.TestDrivers.TlsClientAuth; import com.yahoo.jdisc.service.BindingSetNotFoundException; import com.yahoo.security.KeyUtils; @@ -657,7 +656,7 @@ public class HttpServerTest { assertHttpsRequestTriggersSslHandshakeException( driver, clientCtx, null, null, "Received fatal alert: bad_certificate"); verify(metricConsumer.mockitoMock()) - .add(Metrics.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); assertTrue(driver.close()); } @@ -677,7 +676,7 @@ public class HttpServerTest { assertHttpsRequestTriggersSslHandshakeException( driver, clientCtx, "TLSv1.3", null, "Received fatal alert: protocol_version"); verify(metricConsumer.mockitoMock()) - .add(Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT); + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT); assertTrue(driver.close()); } @@ -697,7 +696,7 @@ public class HttpServerTest { assertHttpsRequestTriggersSslHandshakeException( driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure"); verify(metricConsumer.mockitoMock()) - .add(Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT); + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT); assertTrue(driver.close()); } @@ -721,7 +720,7 @@ public class HttpServerTest { assertHttpsRequestTriggersSslHandshakeException( driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); verify(metricConsumer.mockitoMock()) - .add(Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); assertTrue(driver.close()); } @@ -744,7 +743,7 @@ public class HttpServerTest { assertHttpsRequestTriggersSslHandshakeException( driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); verify(metricConsumer.mockitoMock()) - .add(Metrics.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); assertTrue(driver.close()); } diff --git a/logd/src/tests/empty_forwarder/empty_forwarder_test.cpp b/logd/src/tests/empty_forwarder/empty_forwarder_test.cpp index d1194f30c40..dbc76e694f2 100644 --- a/logd/src/tests/empty_forwarder/empty_forwarder_test.cpp +++ b/logd/src/tests/empty_forwarder/empty_forwarder_test.cpp @@ -11,7 +11,7 @@ using vespalib::metrics::DummyMetricsManager; struct MockMetricsManager : public DummyMetricsManager { int add_count; - MockMetricsManager() : DummyMetricsManager(), add_count(0) {} + MockMetricsManager() noexcept : DummyMetricsManager(), add_count(0) {} void add(Counter::Increment) override { ++add_count; } diff --git a/logd/src/tests/rpc_forwarder/rpc_forwarder_test.cpp b/logd/src/tests/rpc_forwarder/rpc_forwarder_test.cpp index 15a1dc36a87..d39a9ade0a8 100644 --- a/logd/src/tests/rpc_forwarder/rpc_forwarder_test.cpp +++ b/logd/src/tests/rpc_forwarder/rpc_forwarder_test.cpp @@ -85,7 +85,7 @@ make_log_line(const std::string& level, const std::string& payload) struct MockMetricsManager : public DummyMetricsManager { int add_count; - MockMetricsManager() : DummyMetricsManager(), add_count(0) {} + MockMetricsManager() noexcept : DummyMetricsManager(), add_count(0) {} void add(Counter::Increment) override { ++add_count; } diff --git a/logd/src/tests/watcher/watcher_test.cpp b/logd/src/tests/watcher/watcher_test.cpp index 7e7585f9209..f74564b23f4 100644 --- a/logd/src/tests/watcher/watcher_test.cpp +++ b/logd/src/tests/watcher/watcher_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <logd/config_subscriber.h> +#include <vespa/config/common/configcontext.h> #include <logd/watcher.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/threadstackexecutor.h> diff --git a/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java b/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java index 460783457af..9bde5e32fd7 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java @@ -59,10 +59,10 @@ public final class ErrorCode { /** The protocol specified for the message is unknown. */ public static final int UNKNOWN_PROTOCOL = FATAL_ERROR + 7; - /** An error occured while decoding the message. */ + /** An error occurred while decoding the message. */ public static final int DECODE_ERROR = FATAL_ERROR + 8; - /** A timeout occured while sending. */ + /** A timeout occurred while sending. */ public static final int TIMEOUT = FATAL_ERROR + 9; /** The target is running an incompatible version. */ diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Protocol.java b/messagebus/src/main/java/com/yahoo/messagebus/Protocol.java index 3801308d38f..d1c19cda88c 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/Protocol.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Protocol.java @@ -12,37 +12,34 @@ import com.yahoo.messagebus.routing.RoutingPolicy; */ public interface Protocol { - /** - * Returns a global unique name for this protocol. - * - * @return The name. - */ - public String getName(); + /** Returns a global unique name for this protocol. */ + String getName(); /** * Encodes the protocol specific data of a routable into a byte array. * - * @param version The version to encode for. - * @param routable The routable to encode. - * @return The encoded data. + * @param version the version to encode for + * @param routable the routable to encode + * @return the encoded data */ - public byte[] encode(Version version, Routable routable); + byte[] encode(Version version, Routable routable); /** * Decodes the protocol specific data into a routable of the correct type. * - * @param version The version of the serialized routable. - * @param payload The payload to decode from. - * @return The decoded routable. + * @param version the version of the serialized routable + * @param payload the payload to decode from + * @return the decoded routable, or null if it could not be decoded */ - public Routable decode(Version version, byte[] payload); + Routable decode(Version version, byte[] payload); /** * Create a policy of the named type with the named param passed to the constructor of that policy. * - * @param name The name of the policy to create. - * @param param The parameter to that policy's constructor. - * @return The created policy. + * @param name the name of the policy to create + * @param param the parameter to that policy's constructor + * @return the created policy */ - public RoutingPolicy createPolicy(String name, String param); + RoutingPolicy createPolicy(String name, String param); + } diff --git a/messagebus/src/vespa/messagebus/messagebus.cpp b/messagebus/src/vespa/messagebus/messagebus.cpp index c3d6b28b318..ce60f1a3969 100644 --- a/messagebus/src/vespa/messagebus/messagebus.cpp +++ b/messagebus/src/vespa/messagebus/messagebus.cpp @@ -13,7 +13,6 @@ #include <vespa/log/log.h> LOG_SETUP(".messagebus"); -using vespalib::LockGuard; using vespalib::make_string; using namespace std::chrono_literals; @@ -201,7 +200,7 @@ MessageBus::createIntermediateSession(const string &name, IntermediateSession::UP MessageBus::createIntermediateSession(const IntermediateSessionParams ¶ms) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); IntermediateSession::UP ret(new IntermediateSession(*this, params)); _sessions[params.getName()] = ret.get(); if (params.getBroadcastName()) { @@ -224,7 +223,7 @@ MessageBus::createDestinationSession(const string &name, DestinationSession::UP MessageBus::createDestinationSession(const DestinationSessionParams ¶ms) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); DestinationSession::UP ret(new DestinationSession(*this, params)); _sessions[params.getName()] = ret.get(); if (params.getBroadcastName()) { @@ -236,7 +235,7 @@ MessageBus::createDestinationSession(const DestinationSessionParams ¶ms) void MessageBus::unregisterSession(const string &sessionName) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _network.unregisterSession(sessionName); _sessions.erase(sessionName); } @@ -245,7 +244,7 @@ RoutingTable::SP MessageBus::getRoutingTable(const string &protocol) { typedef std::map<string, RoutingTable::SP>::iterator ITR; - LockGuard guard(_lock); + std::lock_guard guard(_lock); ITR itr = _routingTables.find(protocol); if (itr == _routingTables.end()) { return RoutingTable::SP(); // not found @@ -293,7 +292,7 @@ MessageBus::setupRouting(const RoutingSpec &spec) rtm[cfg.getProtocol()] = std::make_shared<RoutingTable>(cfg); } { - LockGuard guard(_lock); + std::lock_guard guard(_lock); std::swap(_routingTables, rtm); } _protocolRepository->clearPolicyCache(); @@ -360,7 +359,7 @@ MessageBus::deliverMessage(Message::UP msg, const string &session) { IMessageHandler *msgHandler = nullptr; { - LockGuard guard(_lock); + std::lock_guard guard(_lock); std::map<string, IMessageHandler*>::iterator it = _sessions.find(session); if (it != _sessions.end()) { msgHandler = it->second; diff --git a/messagebus/src/vespa/messagebus/messagebus.h b/messagebus/src/vespa/messagebus/messagebus.h index c0682967db9..b12054f7006 100644 --- a/messagebus/src/vespa/messagebus/messagebus.h +++ b/messagebus/src/vespa/messagebus/messagebus.h @@ -10,7 +10,6 @@ #include "sourcesession.h" #include <vespa/messagebus/network/inetworkowner.h> #include <vespa/messagebus/routing/routingspec.h> -#include <vespa/vespalib/util/sync.h> #include <map> #include <string> #include <atomic> @@ -38,7 +37,7 @@ class MessageBus : public IMessageHandler, private: using RoutingTableSP = std::shared_ptr<RoutingTable>; INetwork &_network; - vespalib::Lock _lock; + std::mutex _lock; std::map<string, RoutingTableSP> _routingTables; std::map<string, IMessageHandler*> _sessions; std::unique_ptr<ProtocolRepository> _protocolRepository; diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index de3be2ffa01..eb94ab5ff5c 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -96,7 +96,7 @@ RPCNetwork::SendContext::handleVersion(const vespalib::Version *version) { bool shouldSend = false; { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (version == nullptr) { _hasError = true; } else if (*version < _version) { diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index a8eb514387c..2780c3e8770 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -37,7 +37,7 @@ class RPCNetwork : public INetwork, private: using CompressionConfig = vespalib::compression::CompressionConfig; struct SendContext : public RPCTarget::IVersionHandler { - vespalib::Lock _lock; + std::mutex _lock; RPCNetwork &_net; const Message &_msg; uint32_t _traceLevel; @@ -128,7 +128,7 @@ public: * * @param params A complete set of parameters. */ - RPCNetwork(const RPCNetworkParams ¶ms); + explicit RPCNetwork(const RPCNetworkParams ¶ms); /** * Destruct diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.h b/messagebus/src/vespa/messagebus/network/rpctargetpool.h index d47fd977356..bc9e1a4b19f 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.h +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.h @@ -4,7 +4,6 @@ #include "rpcserviceaddress.h" #include "rpctarget.h" #include <vespa/messagebus/itimer.h> -#include <vespa/vespalib/util/sync.h> #include <map> class FRT_Supervisor; diff --git a/messagebus/src/vespa/messagebus/protocolrepository.cpp b/messagebus/src/vespa/messagebus/protocolrepository.cpp index a9891069c44..f23ef0e2ff7 100644 --- a/messagebus/src/vespa/messagebus/protocolrepository.cpp +++ b/messagebus/src/vespa/messagebus/protocolrepository.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "protocolrepository.h" +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".protocolrepository"); @@ -12,7 +13,7 @@ ProtocolRepository::~ProtocolRepository() = default; void ProtocolRepository::clearPolicyCache() { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _routingPolicyCache.clear(); } @@ -66,7 +67,7 @@ ProtocolRepository::getRoutingPolicy(const string &protocolName, { string cacheKey = protocolName; cacheKey.append('.').append(policyName).append(".").append(policyParam); - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); RoutingPolicyCache::iterator cit = _routingPolicyCache.find(cacheKey); if (cit != _routingPolicyCache.end()) { return cit->second; diff --git a/messagebus/src/vespa/messagebus/protocolrepository.h b/messagebus/src/vespa/messagebus/protocolrepository.h index b310ba3586a..28163149e2e 100644 --- a/messagebus/src/vespa/messagebus/protocolrepository.h +++ b/messagebus/src/vespa/messagebus/protocolrepository.h @@ -2,9 +2,9 @@ #pragma once #include "iprotocol.h" -#include <vespa/vespalib/util/sync.h> #include <map> #include <atomic> +#include <mutex> namespace mbus { @@ -19,7 +19,7 @@ private: using ProtocolMap = std::map<string, IProtocol::SP>; using RoutingPolicyCache = std::map<string, IRoutingPolicy::SP>; - vespalib::Lock _lock; // Only guards the cache, + std::mutex _lock; // Only guards the cache, // not the protocols as they are set up during messagebus construction. static constexpr size_t MAX_PROTOCOLS = 16; std::pair<string, std::atomic<IProtocol *>> _protocols[MAX_PROTOCOLS]; diff --git a/messagebus/src/vespa/messagebus/routing/resender.h b/messagebus/src/vespa/messagebus/routing/resender.h index 68b49cde606..f70fd57265c 100644 --- a/messagebus/src/vespa/messagebus/routing/resender.h +++ b/messagebus/src/vespa/messagebus/routing/resender.h @@ -4,7 +4,6 @@ #include "iretrypolicy.h" #include <vespa/messagebus/queue.h> #include <vespa/messagebus/reply.h> -#include <vespa/vespalib/util/sync.h> #include <queue> #include <vector> diff --git a/messagebus/src/vespa/messagebus/routing/routingnode.cpp b/messagebus/src/vespa/messagebus/routing/routingnode.cpp index a47abe185cc..f24afbc07ca 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnode.cpp +++ b/messagebus/src/vespa/messagebus/routing/routingnode.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/messagebus/network/inetwork.h> #include <stack> +#include <cassert> using vespalib::make_string; diff --git a/messagebus/src/vespa/messagebus/routing/routingnode.h b/messagebus/src/vespa/messagebus/routing/routingnode.h index 3902e4ef699..2f82eb0ff92 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnode.h +++ b/messagebus/src/vespa/messagebus/routing/routingnode.h @@ -12,7 +12,6 @@ #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/network/iserviceaddress.h> #include <vespa/messagebus/reply.h> -#include <vespa/vespalib/util/sync.h> #include <vector> #include <map> diff --git a/messagebus/src/vespa/messagebus/sequencer.cpp b/messagebus/src/vespa/messagebus/sequencer.cpp index 2d7a36a28ef..6270284b9e2 100644 --- a/messagebus/src/vespa/messagebus/sequencer.cpp +++ b/messagebus/src/vespa/messagebus/sequencer.cpp @@ -2,6 +2,7 @@ #include "sequencer.h" #include "tracelevel.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> using vespalib::make_string; @@ -37,7 +38,7 @@ Sequencer::filter(Message::UP msg) uint64_t seqId = msg->getSequenceId(); msg->setContext(Context(seqId)); { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); QueueMap::iterator it = _seqMap.find(seqId); if (it != _seqMap.end()) { if (it->second == nullptr) { @@ -85,7 +86,7 @@ Sequencer::handleReply(Reply::UP reply) make_string("Sequencer received reply with sequence id '%" PRIu64 "'.", seq)); Message::UP msg; { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); QueueMap::iterator it = _seqMap.find(seq); MessageQueue *que = it->second; assert(it != _seqMap.end()); @@ -99,7 +100,7 @@ Sequencer::handleReply(Reply::UP reply) que->pop(); } } - if (msg.get() != nullptr) { + if (msg) { sequencedSend(std::move(msg)); } IReplyHandler &handler = reply->getCallStack().pop(*reply); diff --git a/messagebus/src/vespa/messagebus/sequencer.h b/messagebus/src/vespa/messagebus/sequencer.h index 7aae32fc744..54ac4ce5007 100644 --- a/messagebus/src/vespa/messagebus/sequencer.h +++ b/messagebus/src/vespa/messagebus/sequencer.h @@ -2,13 +2,13 @@ #pragma once -#include <map> -#include <vespa/vespalib/util/sync.h> #include "imessagehandler.h" #include "ireplyhandler.h" #include "message.h" #include "reply.h" #include "queue.h" +#include <mutex> +#include <map> namespace mbus { @@ -21,7 +21,7 @@ class Sequencer : public IMessageHandler, public IReplyHandler { private: - vespalib::Lock _lock; + std::mutex _lock; IMessageHandler &_sender; typedef Queue<Message*> MessageQueue; diff --git a/messagebus_test/src/tests/speed/cpp-client.cpp b/messagebus_test/src/tests/speed/cpp-client.cpp index b5829c76c08..2c20b35c597 100644 --- a/messagebus_test/src/tests/speed/cpp-client.cpp +++ b/messagebus_test/src/tests/speed/cpp-client.cpp @@ -17,7 +17,7 @@ using namespace std::chrono_literals; class Client : public IReplyHandler { private: - vespalib::Lock _lock; + std::mutex _lock; uint32_t _okCnt; uint32_t _failCnt; SourceSession::UP _session; @@ -58,7 +58,7 @@ Client::send(uint64_t seq) { void Client::sample(uint32_t &okCnt, uint32_t &failCnt) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); okCnt = _okCnt; failCnt = _failCnt; } @@ -69,7 +69,7 @@ Client::handleReply(Reply::UP reply) { && (reply->getType() == SimpleProtocol::REPLY) && (static_cast<SimpleReply&>(*reply).getValue() == "OK")) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); ++_okCnt; } else { fprintf(stderr, "BAD REPLY\n"); @@ -78,7 +78,7 @@ Client::handleReply(Reply::UP reply) { reply->getError(i).getCode(), reply->getError(i).getMessage().c_str()); } - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); ++_failCnt; } send(); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java index c9d4ad2a5ac..9408c9b917a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java @@ -52,8 +52,10 @@ public class RpcConnector extends AbstractComponent { } public void stop() { - acceptor.shutdown().join(); - supervisor.transport().shutdown().join(); + if (acceptor != null) + acceptor.shutdown().join(); + if (supervisor != null) + supervisor.transport().shutdown().join(); } @Override diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 84e92c2c316..7db9686d5c8 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -13,6 +13,7 @@ #include <vespa/vespalib/stllike/hashtable.hpp> #include <sstream> #include <algorithm> +#include <cassert> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".metrics.manager"); diff --git a/metrics/src/vespa/metrics/metricsnapshot.cpp b/metrics/src/vespa/metrics/metricsnapshot.cpp index 86a33f0993f..f7460c8b6ad 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.cpp +++ b/metrics/src/vespa/metrics/metricsnapshot.cpp @@ -1,8 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "metricsnapshot.h" #include "metricmanager.h" -#include <vespa/log/log.h> +#include <cassert> +#include <vespa/log/log.h> LOG_SETUP(".metrics.snapshot"); namespace metrics { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 6b7b72f2746..f09598a6fb0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -363,8 +363,6 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, "Container should be running with different CPU allocation, wanted: %s, current: %s", wantedContainerResources.toStringCpu(), existingContainer.resources.toStringCpu()); - orchestratorSuspendNode(context); - // Only update CPU resources containerOperations.updateContainer(context, wantedContainerResources.withMemoryBytes(existingContainer.resources.memoryBytes())); return containerOperations.getContainer(context).orElseThrow(() -> diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index fdd950f7428..7880209bbc8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -223,25 +223,25 @@ public class NodeAgentImplTest { ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16); mockGetContainer(dockerImage, resourcesAfterThird, true); - inOrder.verify(orchestrator).suspend(any(String.class)); + inOrder.verify(orchestrator, never()).suspend(any()); inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(resourcesAfterThird)); inOrder.verify(containerOperations, never()).removeContainer(any(), any()); inOrder.verify(containerOperations, never()).startContainer(any()); - inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(orchestrator, never()).resume(any()); // No changes nodeAgent.converge(thirdContext); - inOrder.verify(orchestrator, never()).suspend(any(String.class)); + inOrder.verify(orchestrator, never()).suspend(any()); inOrder.verify(containerOperations, never()).updateContainer(eq(thirdContext), any()); inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator, never()).resume(any(String.class)); + inOrder.verify(orchestrator, never()).resume(any()); // Set the feature flag flagSource.withDoubleFlag(Flags.CONTAINER_CPU_CAP.id(), 2.3); nodeAgent.doConverge(thirdContext); inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(ContainerResources.from(9.2, 4, 16))); - inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(orchestrator, never()).resume(any()); } @Test @@ -645,18 +645,18 @@ public class NodeAgentImplTest { clock.advance(Duration.ofSeconds(31)); nodeAgent.doConverge(context); - inOrder.verify(orchestrator).suspend(hostName); + inOrder.verify(orchestrator, never()).suspend(any()); inOrder.verify(containerOperations).updateContainer(eq(context), eq(ContainerResources.from(0, 2, 16))); inOrder.verify(containerOperations, never()).removeContainer(any(), any()); inOrder.verify(containerOperations, never()).startContainer(any()); - inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(orchestrator, never()).resume(any()); // No changes nodeAgent.converge(context); - inOrder.verify(orchestrator, never()).suspend(any(String.class)); + inOrder.verify(orchestrator, never()).suspend(any()); inOrder.verify(containerOperations, never()).updateContainer(eq(context), any()); inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator, never()).resume(any(String.class)); + inOrder.verify(orchestrator, never()).resume(any()); } @Test diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 455d9e59734..e92f039ad01 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.NodeRepositoryConfig; @@ -502,19 +503,17 @@ public class NodeRepository extends AbstractComponent { } } - public void deactivate(ApplicationId application, NestedTransaction transaction) { - try (Mutex lock = lock(application)) { - deactivate(db.readNodes(application, State.reserved, State.active), transaction); - applications.remove(application, transaction, lock); - } + /** Deactivate nodes owned by application guarded by given lock */ + public void deactivate(NestedTransaction transaction, ProvisionLock lock) { + deactivate(db.readNodes(lock.application(), State.reserved, State.active), transaction, lock); + applications.remove(lock.application(), transaction, lock); } /** - * Deactivates these nodes in a transaction and returns - * the nodes in the new state which will hold if the transaction commits. - * This method does <b>not</b> lock + * Deactivates these nodes in a transaction and returns the nodes in the new state which will hold if the + * transaction commits. */ - public List<Node> deactivate(List<Node> nodes, NestedTransaction transaction) { + public List<Node> deactivate(List<Node> nodes, NestedTransaction transaction, @SuppressWarnings("unused") ProvisionLock lock) { return db.writeTo(State.inactive, nodes, Agent.application, Optional.empty(), transaction); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java index 56e52d9f658..9f45839f1c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; @@ -25,10 +26,7 @@ public class Applications { // read and write all to make sure they are stored in the latest version of the serialized format for (ApplicationId id : ids()) { try (Mutex lock = db.lock(id)) { - // TODO(mpolden): Remove inner lock - try (Mutex innerLock = db.configLock(id)) { - get(id).ifPresent(application -> put(application, lock)); - } + get(id).ifPresent(application -> put(application, lock)); } } } @@ -41,17 +39,19 @@ public class Applications { return db.readApplication(id); } + // TODO: Require ProvisionLock instead of Mutex public void put(Application application, Mutex applicationLock) { NestedTransaction transaction = new NestedTransaction(); put(application, transaction, applicationLock); transaction.commit(); } + // TODO: Require ProvisionLock instead of Mutex public void put(Application application, NestedTransaction transaction, Mutex applicationLock) { db.writeApplication(application, transaction); } - public void remove(ApplicationId application, NestedTransaction transaction, Mutex applicationLock) { + public void remove(ApplicationId application, NestedTransaction transaction, @SuppressWarnings("unused") ProvisionLock lock) { db.deleteApplication(application, transaction); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 267bfefa332..d1d15baa5dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -138,21 +138,20 @@ public class AllocatableClusterResources { Limits applicationLimits, NodeRepository nodeRepository) { var systemLimits = new NodeResourceLimits(nodeRepository); - if ( !exclusive && nodeRepository.zone().getCloud().allowHostSharing()) { // Check if any flavor can fit these hosts + if ( !exclusive && nodeRepository.zone().getCloud().allowHostSharing()) { // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources NodeResources advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources()); advertisedResources = systemLimits.enlargeToLegal(advertisedResources, clusterType); // Attempt to ask for something legal advertisedResources = applicationLimits.cap(advertisedResources); // Overrides other conditions, even if it will then fail NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources); // ... thus, what we really get may change if ( ! systemLimits.isWithinRealLimits(realResources, clusterType)) return Optional.empty(); - for (Flavor flavor : nodeRepository.flavors().getFlavors()) { - if (flavor.resources().satisfies(advertisedResources)) + if (matchesAny(nodeRepository.flavors().getFlavors(), advertisedResources)) return Optional.of(new AllocatableClusterResources(wantedResources.with(realResources), advertisedResources, wantedResources.nodeResources(), clusterType)); - } - return Optional.empty(); + else + return Optional.empty(); } else { // Return the cheapest flavor satisfying the requested resources, if any NodeResources cappedWantedResources = applicationLimits.cap(wantedResources.nodeResources()); @@ -185,6 +184,14 @@ public class AllocatableClusterResources { } } + /** Returns true if the given resources could be allocated on any of the given flavors */ + private static boolean matchesAny(List<Flavor> flavors, NodeResources advertisedResources) { + // Tenant nodes should not consume more than half the resources of the biggest hosts + // to make it easier to shift them between hosts. + return flavors.stream().anyMatch(flavor -> flavor.resources().withVcpu(flavor.resources().vcpu() / 2) + .satisfies(advertisedResources)); + } + private static boolean between(NodeResources min, NodeResources max, NodeResources r) { if ( ! min.isUnspecified() && ! min.justNonNumbers().compatibleWith(r.justNonNumbers())) return false; if ( ! max.isUnspecified() && ! max.justNonNumbers().compatibleWith(r.justNonNumbers())) return false; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index d359b86ae85..a267d59f1dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.provision.Node; @@ -9,13 +8,8 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; /** * The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 55d17f3e080..ca2a59ca6d4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -22,7 +22,7 @@ import java.util.logging.Level; import java.util.stream.Collectors; /** - * Periodically expire load balancers. + * Periodically expire load balancers and de-provision inactive ones. * * Load balancers expire from the following states: * @@ -123,14 +123,11 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { /** Apply operation to all load balancers that exist in given state, while holding lock */ private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) { for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lock(id.application())) { - // TODO(mpolden): Remove inner lock - try (var innerLock = db.configLock(id.application())) { - var loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop - if (loadBalancer.get().state() != state) continue; // Wrong state - operation.accept(loadBalancer.get()); - } + try (var lock = db.lock(id.application(), Duration.ofSeconds(1))) { + var loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop + if (loadBalancer.get().state() != state) continue; // Wrong state + operation.accept(loadBalancer.get()); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 1cf1bb91d0c..160e448aeab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -244,6 +244,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { metric.set("lockAttempt.locked", lockMetrics.getAndResetAcquireSucceededCount(), context); metric.set("lockAttempt.release", lockMetrics.getAndResetReleaseCount(), context); metric.set("lockAttempt.releaseFailed", lockMetrics.getAndResetReleaseFailedCount(), context); + metric.set("lockAttempt.reentry", lockMetrics.getAndResetReentryCount(), context); setLockLatencyMetrics("acquire", lockMetrics.getAndResetAcquireLatencyMetrics(), context); setLockLatencyMetrics("locked", lockMetrics.getAndResetLockedLatencyMetrics(), context); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 2ce3fa6c0f6..38511db6c4d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.HostLivenessTracker; @@ -199,7 +199,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { // Lock and update status ApplicationId owner = node.get().allocation().get().owner(); - try (var lock = nodeRepository().lock(owner, Duration.ofSeconds(1))) { + try (var lock = nodeRepository().lock(owner)) { node = getNode(hostname.toString(), owner, lock); // Re-get inside lock if (node.isEmpty()) return; // Node disappeared or changed allocation if (badNode) { @@ -207,8 +207,9 @@ public class NodeFailer extends NodeRepositoryMaintainer { } else { clearDownRecord(node.get(), lock); } - } catch (UncheckedTimeoutException ignored) { - // Fine, we'll try updating this node in the next run + } catch (ApplicationLockException e) { + // Fine, carry on with other nodes. We'll try updating this one in the next run + log.log(Level.WARNING, "Could not lock " + owner + ": " + Exceptions.toMessageString(e)); } }); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 2c00b4bab68..91c683d139e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -61,7 +61,6 @@ public class CuratorDatabaseClient { private static final Path root = Path.fromString("/provision/v1"); private static final Path lockPath = root.append("locks"); - private static final Path configLockPath = Path.fromString("/config/v2/locks/"); private static final Path loadBalancersPath = root.append("loadBalancers"); private static final Path applicationsPath = root.append("applications"); private static final Path inactiveJobsPath = root.append("inactiveJobs"); @@ -328,15 +327,6 @@ public class CuratorDatabaseClient { return lockPath; } - /** Creates and returns the config lock path for this application */ - // TODO(mpolden): Remove - private Path configLockPath(ApplicationId application) { - // This must match the lock path used by com.yahoo.vespa.config.server.application.TenantApplications - Path lockPath = configLockPath.append(application.tenant().value()).append(application.serializedForm()); - db.create(lockPath); - return lockPath; - } - private String toDir(Node.State state) { switch (state) { case active: return "allocated"; // legacy name @@ -372,20 +362,6 @@ public class CuratorDatabaseClient { } } - // TODO(mpolden): Remove - private Lock configLock(ApplicationId application, Duration timeout) { - try { - return db.lock(configLockPath(application), timeout); - } catch (UncheckedTimeoutException e) { - throw new ApplicationLockException(e); - } - } - - // TODO(mpolden): Remove - public Lock configLock(ApplicationId application) { - return configLock(application, defaultLockTimeout); - } - // Applications ----------------------------------------------------------- public List<ApplicationId> readApplicationIds() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index e36d5fa4075..b85862446a8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ParentHostUnavailableException; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -37,12 +38,10 @@ class Activator { this.loadBalancerProvisioner = loadBalancerProvisioner; } - /** Activate required resources for given application */ - public void activate(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction) { - try (Mutex lock = nodeRepository.lock(application)) { - activateNodes(application, hosts, transaction, lock); - activateLoadBalancers(application, hosts, transaction, lock); - } + /** Activate required resources for application guarded by given lock */ + public void activate(Collection<HostSpec> hosts, NestedTransaction transaction, ProvisionLock lock) { + activateNodes(hosts, transaction, lock); + activateLoadBalancers(hosts, transaction, lock); } /** @@ -55,12 +54,11 @@ class Activator { * Nodes in active which are not present in <code>hosts</code> are moved to inactive. * * @param transaction Transaction with operations to commit together with any operations done within the repository. - * @param application the application to allocate nodes for * @param hosts the hosts to make the set of active nodes of this - * @param applicationLock application lock that must be held when calling this + * @param lock provision lock that must be held when calling this */ - private void activateNodes(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction, - @SuppressWarnings("unused") Mutex applicationLock) { + private void activateNodes(Collection<HostSpec> hosts, NestedTransaction transaction, ProvisionLock lock) { + ApplicationId application = lock.application(); Set<String> hostnames = hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); NodeList allNodes = nodeRepository.list(); NodeList applicationNodes = allNodes.owner(application); @@ -84,7 +82,7 @@ class Activator { List<Node> activeToRemove = removeHostsFromList(hostnames, active); activeToRemove = activeToRemove.stream().map(Node::unretire).collect(Collectors.toList()); // only active nodes can be retired - nodeRepository.deactivate(activeToRemove, transaction); + nodeRepository.deactivate(activeToRemove, transaction, lock); nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction); unreserveParentsOf(reservedToActivate); @@ -106,9 +104,8 @@ class Activator { } /** Activate load balancers */ - private void activateLoadBalancers(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction, - @SuppressWarnings("unused") Mutex applicationLock) { - loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(application, allClustersOf(hosts), applicationLock, transaction)); + private void activateLoadBalancers(Collection<HostSpec> hosts, NestedTransaction transaction, ProvisionLock lock) { + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, allClustersOf(hosts), lock)); } private static Set<ClusterSpec> allClustersOf(Collection<HostSpec> hosts) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index e6ace49cacd..b3506a0c102 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; -import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -21,6 +20,7 @@ import com.yahoo.vespa.service.monitor.InfraApplicationApi; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -97,7 +97,7 @@ public class InfraDeployerImpl implements InfraDeployer { @Override public long activate() { - try (Mutex lock = nodeRepository.lock(application.getApplicationId())) { + try (var lock = provisioner.lock(application.getApplicationId())) { prepare(); if (hostSpecs.isEmpty()) { @@ -105,7 +105,7 @@ public class InfraDeployerImpl implements InfraDeployer { removeApplication(application.getApplicationId()); } else { NestedTransaction nestedTransaction = new NestedTransaction(); - provisioner.activate(nestedTransaction, application.getApplicationId(), hostSpecs); + provisioner.activate(nestedTransaction, hostSpecs, lock); nestedTransaction.commit(); duperModel.infraApplicationActivated( @@ -128,10 +128,12 @@ public class InfraDeployerImpl implements InfraDeployer { private void removeApplication(ApplicationId applicationId) { // Use the DuperModel as source-of-truth on whether it has also been activated (to avoid periodic removals) if (duperModel.infraApplicationIsActive(applicationId)) { - NestedTransaction nestedTransaction = new NestedTransaction(); - provisioner.remove(nestedTransaction, applicationId); - nestedTransaction.commit(); - duperModel.infraApplicationRemoved(applicationId); + try (var lock = provisioner.lock(applicationId)) { + NestedTransaction nestedTransaction = new NestedTransaction(); + provisioner.remove(nestedTransaction, lock); + nestedTransaction.commit(); + duperModel.infraApplicationRemoved(applicationId); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 09a33093654..634726f9d71 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -5,8 +5,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.exception.LoadBalancerServiceException; -import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; @@ -60,11 +60,8 @@ public class LoadBalancerProvisioner { // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { - // TODO(mpolden): Remove inner lock - try (var innerLock = db.configLock(id.application())) { - var loadBalancer = db.readLoadBalancer(id); - loadBalancer.ifPresent(db::writeLoadBalancer); - } + var loadBalancer = db.readLoadBalancer(id); + loadBalancer.ifPresent(db::writeLoadBalancer); } } } @@ -83,12 +80,9 @@ public class LoadBalancerProvisioner { if (!canForwardTo(requestedNodes.type(), cluster)) return; // Nothing to provision for this node and cluster type if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lock(application)) { - // TODO(mpolden): Remove inner lock - try (var innerLock = db.configLock(application)) { - ClusterSpec.Id clusterId = effectiveId(cluster); - List<Node> nodes = nodesOf(clusterId, application); - provision(application, clusterId, nodes, false, lock); - } + ClusterSpec.Id clusterId = effectiveId(cluster); + List<Node> nodes = nodesOf(clusterId, application); + provision(application, clusterId, nodes, false, new ProvisionLock(application, lock)); } } @@ -102,35 +96,24 @@ public class LoadBalancerProvisioner { * * Calling this when no load balancer has been prepared for given cluster is a no-op. */ - public void activate(ApplicationId application, Set<ClusterSpec> clusters, - @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { - try (var lock = db.lock(application)) { - // TODO(mpolden): Remove inner lock - try (var innerLock = db.configLock(application)) { - for (var cluster : loadBalancedClustersOf(application).entrySet()) { - // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, cluster.getKey(), cluster.getValue(), true, lock); - } - // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(application, clusters.stream() - .map(LoadBalancerProvisioner::effectiveId) - .collect(Collectors.toSet())); - deactivate(surplusLoadBalancers, transaction); - } + public void activate(NestedTransaction transaction, Set<ClusterSpec> clusters, ProvisionLock lock) { + for (var cluster : loadBalancedClustersOf(lock.application()).entrySet()) { + // Provision again to ensure that load balancer instance is re-configured with correct nodes + provision(lock.application(), cluster.getKey(), cluster.getValue(), true, lock); } + // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed + var surplusLoadBalancers = surplusLoadBalancersOf(lock.application(), clusters.stream() + .map(LoadBalancerProvisioner::effectiveId) + .collect(Collectors.toSet())); + deactivate(surplusLoadBalancers, transaction); } /** * Deactivate all load balancers assigned to given application. This is a no-op if an application does not have any * load balancer(s). */ - public void deactivate(ApplicationId application, NestedTransaction transaction) { - try (var lock = nodeRepository.lock(application)) { - // TODO(mpolden): Remove inner lock - try (var innerLock = db.configLock(application)) { - deactivate(nodeRepository.loadBalancers(application).asList(), transaction); - } - } + public void deactivate(NestedTransaction transaction, ProvisionLock lock) { + deactivate(nodeRepository.loadBalancers(lock.application()).asList(), transaction); } /** Returns load balancers of given application that are no longer referenced by given clusters */ @@ -168,7 +151,7 @@ public class LoadBalancerProvisioner { /** Idempotently provision a load balancer for given application and cluster */ private void provision(ApplicationId application, ClusterSpec.Id clusterId, List<Node> nodes, boolean activate, - @SuppressWarnings("unused") Mutex loadBalancersLock) { + @SuppressWarnings("unused") ProvisionLock lock) { var id = new LoadBalancerId(application, clusterId); var now = nodeRepository.clock().instant(); var loadBalancer = db.readLoadBalancer(id); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index aea85729d6d..b79b87ae86c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; @@ -118,9 +119,17 @@ public class NodeRepositoryProvisioner implements Provisioner { } @Override + // TODO(mpolden): Remove public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { + try (var lock = lock(application)) { + activate(transaction, hosts, lock); + } + } + + @Override + public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) { validate(hosts); - activator.activate(application, hosts, transaction); + activator.activate(hosts, transaction, lock); } @Override @@ -129,9 +138,22 @@ public class NodeRepositoryProvisioner implements Provisioner { } @Override + // TODO(mpolden): Remove public void remove(NestedTransaction transaction, ApplicationId application) { - nodeRepository.deactivate(application, transaction); - loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(application, transaction)); + try (var lock = lock(application)) { + remove(transaction, lock); + } + } + + @Override + public void remove(NestedTransaction transaction, ProvisionLock lock) { + nodeRepository.deactivate(transaction, lock); + loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(transaction, lock)); + } + + @Override + public ProvisionLock lock(ApplicationId application) { + return new ProvisionLock(application, nodeRepository.lock(application)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 9b498677827..e84a43cd1aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -61,6 +61,7 @@ public class LocksResponse extends HttpResponse { lockPathCursor.setLong("lockedCount", lockMetrics.getCumulativeAcquireSucceededCount()); lockPathCursor.setLong("releaseCount", lockMetrics.getCumulativeReleaseCount()); lockPathCursor.setLong("releaseFailedCount", lockMetrics.getCumulativeReleaseFailedCount()); + lockPathCursor.setLong("reentryCount", lockMetrics.getCumulativeReentryCount()); setLatency(lockPathCursor, "acquire", lockMetrics.getAcquireLatencyMetrics()); setLatency(lockPathCursor, "locked", lockMetrics.getLockedLatencyMetrics()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index 7f613c85cf5..4baaeb12167 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -155,10 +155,12 @@ public class MockDeployer implements Deployer { prepare(); if (failActivate) throw new IllegalStateException("failActivate is true"); - try (NestedTransaction t = new NestedTransaction()) { - provisioner.activate(t, application.id(), preparedHosts); - t.commit(); - lastDeployTimes.put(application.id, clock.instant()); + try (var lock = provisioner.lock(application.id)) { + try (NestedTransaction t = new NestedTransaction()) { + provisioner.activate(t, preparedHosts, lock); + t.commit(); + lastDeployTimes.put(application.id, clock.instant()); + } } return redeployments++; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index e3a3c0f4fce..042b4aa049c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -197,9 +197,11 @@ public class MockNodeRepository extends NodeRepository { } private void activate(List<HostSpec> hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) { - NestedTransaction transaction = new NestedTransaction(); - provisioner.activate(transaction, application, hosts); - transaction.commit(); + try (var lock = provisioner.lock(application)) { + NestedTransaction transaction = new NestedTransaction(); + provisioner.activate(transaction, hosts, lock); + transaction.commit(); + } } private void addRecord(String name, String ipAddress) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java index 12731e30f46..3dc96e0011b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java @@ -6,12 +6,12 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.transaction.NestedTransaction; import java.util.Collection; -import java.util.Collections; import java.util.List; /** @@ -21,7 +21,7 @@ public class MockProvisioner implements Provisioner { @Override public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { - return Collections.emptyList(); + return List.of(); } @Override @@ -30,13 +30,28 @@ public class MockProvisioner implements Provisioner { } @Override + public void activate(NestedTransaction transaction, Collection<HostSpec> hosts, ProvisionLock lock) { + + } + + @Override public void remove(NestedTransaction transaction, ApplicationId application) { } @Override + public void remove(NestedTransaction transaction, ProvisionLock lock) { + + } + + @Override public void restart(ApplicationId application, HostFilter filter) { } + @Override + public ProvisionLock lock(ApplicationId application) { + return null; + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java index bdb14a868bd..ea304233f8a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; import org.junit.Test; @@ -30,7 +31,7 @@ public class ApplicationsTest { assertEquals(app1, applications.get(app1).get().id()); assertEquals(List.of(app1), applications.ids()); NestedTransaction t = new NestedTransaction(); - applications.remove(app1, t, tester.nodeRepository().lock(app1)); + applications.remove(app1, t, provisionLock(app1, tester)); t.commit(); assertTrue(applications.get(app1).isEmpty()); assertEquals(List.of(), applications.ids()); @@ -43,13 +44,17 @@ public class ApplicationsTest { t.commit(); assertEquals(List.of(app1, app2, app3), applications.ids()); t = new NestedTransaction(); - applications.remove(app1, t, tester.nodeRepository().lock(app1)); - applications.remove(app2, t, tester.nodeRepository().lock(app2)); - applications.remove(app3, t, tester.nodeRepository().lock(app3)); + applications.remove(app1, t, provisionLock(app1, tester)); + applications.remove(app2, t, provisionLock(app2, tester)); + applications.remove(app3, t, provisionLock(app3, tester)); assertEquals(List.of(app1, app2, app3), applications.ids()); t.commit(); assertTrue(applications.get(app1).isEmpty()); assertEquals(List.of(), applications.ids()); } + private ProvisionLock provisionLock(ApplicationId application, NodeRepositoryTester tester) { + return new ProvisionLock(application, tester.nodeRepository().lock(application)); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 5b4b0b5bcb2..d615d418c50 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -78,7 +78,7 @@ public class AutoscalingTest { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); - AutoscalingTester tester = new AutoscalingTester(resources); + AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); ApplicationId application1 = tester.applicationId("application1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); @@ -131,7 +131,7 @@ public class AutoscalingTest { @Test public void autoscaling_respects_upper_limit() { - NodeResources hostResources = new NodeResources(3, 100, 100, 1); + NodeResources hostResources = new NodeResources(6, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); AutoscalingTester tester = new AutoscalingTester(hostResources); @@ -155,7 +155,7 @@ public class AutoscalingTest { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 4, 1, new NodeResources(1.8, 7.4, 8.5, 1)); ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); - AutoscalingTester tester = new AutoscalingTester(resources); + AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); ApplicationId application1 = tester.applicationId("application1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); @@ -209,7 +209,7 @@ public class AutoscalingTest { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = min; - AutoscalingTester tester = new AutoscalingTester(resources); + AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); ApplicationId application1 = tester.applicationId("application1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); @@ -227,7 +227,7 @@ public class AutoscalingTest { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(20, 20, new NodeResources(100, 1000, 1000, 1)); - AutoscalingTester tester = new AutoscalingTester(resources); + AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); ApplicationId application1 = tester.applicationId("application1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); @@ -245,7 +245,7 @@ public class AutoscalingTest { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); - AutoscalingTester tester = new AutoscalingTester(resources); + AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); ApplicationId application1 = tester.applicationId("application1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); @@ -278,7 +278,7 @@ public class AutoscalingTest { @Test public void autoscaling_avoids_illegal_configurations() { - NodeResources hostResources = new NodeResources(3, 100, 100, 1); + NodeResources hostResources = new NodeResources(6, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); AutoscalingTester tester = new AutoscalingTester(hostResources); @@ -287,7 +287,7 @@ public class AutoscalingTest { ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); // deploy - tester.deploy(application1, cluster1, 6, 1, hostResources); + tester.deploy(application1, cluster1, 6, 1, hostResources.withVcpu(hostResources.vcpu() / 2)); tester.addMeasurements(Resource.memory, 0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", 6, 1, 2.8, 4.0, 95.0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index a817cc4f413..258e2c7cd42 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -338,9 +338,11 @@ public class FailedExpirerTest { public FailureScenario allocate(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity capacity) { List<HostSpec> preparedNodes = provisioner.prepare(applicationId, clusterSpec, capacity, (level, message) -> System.out.println(level + ": " + message) ); - NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); - provisioner.activate(transaction, applicationId, Set.copyOf(preparedNodes)); - transaction.commit(); + try (var lock = provisioner.lock(applicationId)) { + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, Set.copyOf(preparedNodes), lock); + transaction.commit(); + } return this; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index 4185ff2c25c..de0e5d4223a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -55,7 +54,7 @@ public class LoadBalancerExpirerTest { assertEquals(3, loadBalancers.get().size()); // Remove one application deactivates load balancers for that application - removeApplication(app1); + tester.remove(app1); assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb1).state()); assertNotSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb2).state()); @@ -146,12 +145,6 @@ public class LoadBalancerExpirerTest { Agent.system, this.getClass().getSimpleName()); } - private void removeApplication(ApplicationId application) { - NestedTransaction transaction = new NestedTransaction(); - tester.provisioner().remove(transaction, application); - transaction.commit(); - } - private void deployApplication(ApplicationId application, ClusterSpec.Id... clusters) { deployApplication(application, true, clusters); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 4d79726fd19..5d5c447ef74 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -159,6 +159,7 @@ public class MetricsReporterTest { verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.locked", 3); verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.release", 3); verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.releaseFailed", 0); + verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.reentry", 0); metric.remove("lockAttempt.acquireLatency"); metric.remove("lockAttempt.acquireMaxActiveLatency"); metric.remove("lockAttempt.acquireHz"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 84b04e040e7..250555ae4fb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -265,9 +265,11 @@ public class NodeFailTester { private void activate(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, capacity, null); - NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); - provisioner.activate(transaction, applicationId, hosts); - transaction.commit(); + try (var lock = provisioner.lock(applicationId)) { + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, hosts, lock); + transaction.commit(); + } } /** Returns the node with the highest membership index from the given set of allocated nodes */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index 65b79c2df04..390c1213718 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; @@ -81,7 +82,7 @@ public class RebalancerTest { // --- Making the system stable enables rebalancing NestedTransaction tx = new NestedTransaction(); - tester.nodeRepository().deactivate(List.of(cpuSkewedNode), tx); + tester.nodeRepository().deactivate(List.of(cpuSkewedNode), tx, new ProvisionLock(cpuApp, () -> {})); tx.commit(); // ... if activation fails when trying, we clean up the state diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 6062b501513..8de769ae113 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -183,9 +183,11 @@ public class RetiredExpirerTest { private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups, NodeRepositoryProvisioner provisioner) { List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.from(new ClusterResources(nodes, groups, nodeResources)), null); - NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); - provisioner.activate(transaction, applicationId, hosts); - transaction.commit(); + try (var lock = provisioner.lock(applicationId)) { + NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); + provisioner.activate(transaction, hosts, lock); + transaction.commit(); + } } private void createReadyNodes(int count, NodeResources nodeResources, NodeRepository nodeRepository) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index edbbb3ed2e3..4fae7cf0ab9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -24,7 +25,9 @@ import com.yahoo.vespa.service.monitor.InfraApplicationApi; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.mockito.ArgumentMatcher; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.Set; @@ -98,10 +101,14 @@ public class InfraDeployerImplTest { verify(duperModelInfraApi, never()).infraApplicationActivated(any(), any()); if (applicationIsActive) { verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId()); - verify(provisioner).remove(any(), eq(application.getApplicationId())); + ArgumentMatcher<ProvisionLock> lockMatcher = lock -> { + assertEquals(application.getApplicationId(), lock.application()); + return true; + }; + verify(provisioner).remove(any(), argThat(lockMatcher)); verify(duperModelInfraApi).infraApplicationRemoved(eq(application.getApplicationId())); } else { - verify(provisioner, never()).remove(any(), any()); + verify(provisioner, never()).remove(any(), any(ProvisionLock.class)); verify(duperModelInfraApi, never()).infraApplicationRemoved(any()); } } @@ -129,10 +136,15 @@ public class InfraDeployerImplTest { private void verifyActivated(String... hostnames) { verify(duperModelInfraApi).infraApplicationActivated( eq(application.getApplicationId()), eq(Stream.of(hostnames).map(HostName::from).collect(Collectors.toList()))); - verify(provisioner).activate(any(), eq(application.getApplicationId()), argThat(hostSpecs -> { + ArgumentMatcher<ProvisionLock> lockMatcher = lock -> { + assertEquals(application.getApplicationId(), lock.application()); + return true; + }; + ArgumentMatcher<Collection<HostSpec>> hostsMatcher = hostSpecs -> { assertEquals(Set.of(hostnames), hostSpecs.stream().map(HostSpec::hostname).collect(Collectors.toSet())); return true; - })); + }; + verify(provisioner).activate(any(), argThat(hostsMatcher), argThat(lockMatcher)); } private Node addNode(int id, Node.State state, Optional<Version> wantedVespaVersion) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 3d784c403f3..735ef2f0fcb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -6,12 +6,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; @@ -22,7 +20,6 @@ import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import org.junit.Test; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; @@ -136,9 +133,7 @@ public class LoadBalancerProvisionerTest { .get()); // Application is removed, nodes and load balancer are deactivated - NestedTransaction removeTransaction = new NestedTransaction(); - tester.provisioner().remove(removeTransaction, app1); - removeTransaction.commit(); + tester.remove(app1); dirtyNodesOf(app1); assertTrue("No nodes are allocated to " + app1, tester.nodeRepository().getNodes(app1, Node.State.reserved, Node.State.active).isEmpty()); assertEquals(2, lbApp1.get().size()); @@ -170,9 +165,7 @@ public class LoadBalancerProvisionerTest { assertFalse("Load balancer is reconfigured with reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty()); // Application is removed, nodes are deleted and load balancer is deactivated - NestedTransaction removeTransaction = new NestedTransaction(); - tester.provisioner().remove(removeTransaction, app1); - removeTransaction.commit(); + tester.remove(app1); tester.nodeRepository().database().removeNodes(tester.nodeRepository().getNodes(NodeType.tenant)); assertTrue("Nodes are deleted", tester.nodeRepository().getNodes(NodeType.tenant).isEmpty()); assertSame("Load balancer is deactivated", LoadBalancer.State.inactive, lb.get().state()); @@ -260,21 +253,6 @@ public class LoadBalancerProvisionerTest { return allNodes; } - private void makeDynamicDockerNodes(int n, NodeType nodeType) { - tester.makeReadyHosts(n, new NodeResources(1, 4, 10, 0.3)); - List<Node> nodes = new ArrayList<>(n); - for (int i = 1; i <= n; i++) { - var node = Node.createDockerNode(Set.of(), "vnode" + i, - tester.nodeRepository().getNodes(NodeType.host).get(n - 1).hostname(), - new NodeResources(1, 4, 10, 0.3), - nodeType); - nodes.add(node); - } - nodes = tester.nodeRepository().database().addNodesInState(nodes, Node.State.reserved, Agent.system); - nodes = tester.nodeRepository().setDirty(nodes, Agent.system, getClass().getSimpleName()); - tester.nodeRepository().setReady(nodes, Agent.system, getClass().getSimpleName()); - } - private void assignIps(List<Node> nodes) { try (var lock = tester.nodeRepository().lockUnallocated()) { for (int i = 0; i < nodes.size(); i++) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 1c03bbc6a56..567ec78576c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -98,9 +98,7 @@ public class ProvisioningTest { assertEquals(5, tester.getNodes(application1, Node.State.inactive).size()); // delete app - NestedTransaction removeTransaction = new NestedTransaction(); - tester.provisioner().remove(removeTransaction, application1); - removeTransaction.commit(); + tester.remove(application1); assertEquals(tester.toHostNames(state1.allHosts), tester.toHostNames(tester.nodeRepository().getNodes(application1, Node.State.inactive))); assertEquals(0, tester.getNodes(application1, Node.State.active).size()); @@ -250,9 +248,7 @@ public class ProvisioningTest { SystemState state7 = prepare(application1, 8, 2, 2, 2, defaultResources, tester); // delete app - NestedTransaction removeTransaction = new NestedTransaction(); - tester.provisioner().remove(removeTransaction, application1); - removeTransaction.commit(); + tester.remove(application1); assertEquals(0, tester.getNodes(application1, Node.State.active).size()); assertEquals(0, tester.getNodes(application1, Node.State.reserved).size()); } @@ -600,9 +596,7 @@ public class ProvisioningTest { SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); // Simulate expiry - NestedTransaction deactivateTransaction = new NestedTransaction(); - tester.nodeRepository().deactivate(application, deactivateTransaction); - deactivateTransaction.commit(); + tester.deactivate(application); try { tester.activate(application, state.allHosts); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index a176b28182e..6b7b979e083 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -17,6 +17,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -203,14 +204,25 @@ public class ProvisioningTester { } public Collection<HostSpec> activate(ApplicationId application, Collection<HostSpec> hosts) { - NestedTransaction transaction = new NestedTransaction(); - transaction.add(new CuratorTransaction(curator)); - provisioner.activate(transaction, application, hosts); - transaction.commit(); + try (var lock = provisioner.lock(application)) { + NestedTransaction transaction = new NestedTransaction(); + transaction.add(new CuratorTransaction(curator)); + provisioner.activate(transaction, hosts, lock); + transaction.commit(); + } assertEquals(toHostNames(hosts), toHostNames(nodeRepository.getNodes(application, Node.State.active))); return hosts; } + /** Remove all resources allocated to application */ + public void remove(ApplicationId application) { + try (var lock = provisioner.lock(application)) { + NestedTransaction transaction = new NestedTransaction(); + provisioner.remove(transaction, lock); + transaction.commit(); + } + } + public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString())).vespaVersion(version).build(); Capacity capacity = Capacity.fromRequiredNodeType(nodeType); @@ -223,9 +235,11 @@ public class ProvisioningTester { } public void deactivate(ApplicationId applicationId) { - NestedTransaction deactivateTransaction = new NestedTransaction(); - nodeRepository.deactivate(applicationId, deactivateTransaction); - deactivateTransaction.commit(); + try (var lock = nodeRepository.lock(applicationId)) { + NestedTransaction deactivateTransaction = new NestedTransaction(); + nodeRepository.deactivate(deactivateTransaction, new ProvisionLock(applicationId, lock)); + deactivateTransaction.commit(); + } } public Collection<String> toHostNames(Collection<HostSpec> hosts) { diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 297a3319939..3285e03db67 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -12,6 +12,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <algorithm> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".dummypersistence"); diff --git a/persistence/src/vespa/persistence/spi/bucket.h b/persistence/src/vespa/persistence/spi/bucket.h index 175aba376a9..b7d04150108 100644 --- a/persistence/src/vespa/persistence/spi/bucket.h +++ b/persistence/src/vespa/persistence/spi/bucket.h @@ -36,7 +36,7 @@ public: /** Convert easily to a document bucket id to make class easy to use. */ operator document::BucketId() const { return _bucket.getBucketId(); } - bool operator==(const Bucket& o) const { + bool operator==(const Bucket& o) const noexcept { return (_bucket == o._bucket && _partition == o._partition); } diff --git a/persistencetypes/src/persistence/spi/types.h b/persistencetypes/src/persistence/spi/types.h index c11f3dacf06..59891e04710 100644 --- a/persistencetypes/src/persistence/spi/types.h +++ b/persistencetypes/src/persistence/spi/types.h @@ -29,10 +29,10 @@ namespace document { typedef type Type; \ name() noexcept : _value() {} \ explicit name(type v) noexcept : _value(v) {} \ - operator type() const { return _value; } \ - operator type&() { return _value; } \ - type getValue() const { return _value; } \ - name& operator=(type val) { _value = val; return *this; } \ + operator type() const noexcept { return _value; } \ + operator type&() noexcept { return _value; } \ + type getValue() const noexcept { return _value; } \ + name& operator=(type val) noexcept { _value = val; return *this; } \ friend vespalib::nbostream & \ operator<<(vespalib::nbostream &os, const name &wrapped); \ friend vespalib::nbostream & \ diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h index 71319d782f1..c26906ac7c0 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h @@ -36,8 +36,8 @@ private: int32_t _weight; public: - WeightedType() : _value(T()), _weight(1) { } - WeightedType(T value_, int32_t weight_ = 1) : _value(value_), _weight(weight_) { } + WeightedType() noexcept : _value(T()), _weight(1) { } + WeightedType(T value_, int32_t weight_ = 1) noexcept : _value(value_), _weight(weight_) { } const T & getValue() const { return _value; } const T & value() const { return _value; } void setValue(const T & v) { _value = v; } diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 1d492cb558f..533e8881d27 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -7,6 +7,7 @@ #include <vespa/config-rank-profiles.h> #include <vespa/config/config.h> #include <vespa/config/helper/legacy.h> +#include <vespa/config/common/configcontext.h> #include <vespa/config/common/exceptions.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_cache/constant_value.h> @@ -186,7 +187,7 @@ App::Main() bool ok = false; try { - IConfigContext::SP ctx(new ConfigContext(*config::legacyConfigId2Spec(configid))); + auto ctx = std::make_shared<ConfigContext>(*config::legacyConfigId2Spec(configid)); vespalib::string cfgId(config::legacyConfigId2ConfigId(configid)); ConfigSubscriber subscriber(ctx); ConfigHandle<VerifyRanksetupConfig>::UP myHandle = subscriber.subscribe<VerifyRanksetupConfig>(cfgId); diff --git a/searchcore/src/apps/vespa-feed-bm/bm_storage_link_context.h b/searchcore/src/apps/vespa-feed-bm/bm_storage_link_context.h index adb2a13ec10..f2df20f1f66 100644 --- a/searchcore/src/apps/vespa-feed-bm/bm_storage_link_context.h +++ b/searchcore/src/apps/vespa-feed-bm/bm_storage_link_context.h @@ -10,7 +10,7 @@ class BmStorageLink; struct BmStorageLinkContext { BmStorageLink* bm_link; - BmStorageLinkContext() + BmStorageLinkContext() noexcept : bm_link(nullptr) { } diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index 585a493559d..dc2dca4d9c8 100644 --- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -24,6 +24,7 @@ #include <vespa/config-summary.h> #include <vespa/config-summarymap.h> #include <vespa/config-upgrading.h> +#include <vespa/config/common/configcontext.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/repo/configbuilder.h> diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 8711a21e5e6..a647085e3d1 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -739,24 +739,31 @@ assertPutDone(AttributeVector &attr, int32_t expVal) void putAttributes(AttributeWriterTest &t, std::vector<uint32_t> expExecuteHistory) { + // Since executor distribution depends on the unspecified hash function in vespalib, + // decouple attribute names from their usage to allow for picking names that hash + // more evenly for a particular implementation. + vespalib::string a1_name = "a1"; + vespalib::string a2_name = "a2x"; + vespalib::string a3_name = "a3y"; + Schema s; - s.addAttributeField(Schema::AttributeField("a1", schema::DataType::INT32, CollectionType::SINGLE)); - s.addAttributeField(Schema::AttributeField("a2", schema::DataType::INT32, CollectionType::SINGLE)); - s.addAttributeField(Schema::AttributeField("a3", schema::DataType::INT32, CollectionType::SINGLE)); + s.addAttributeField(Schema::AttributeField(a1_name, schema::DataType::INT32, CollectionType::SINGLE)); + s.addAttributeField(Schema::AttributeField(a2_name, schema::DataType::INT32, CollectionType::SINGLE)); + s.addAttributeField(Schema::AttributeField(a3_name, schema::DataType::INT32, CollectionType::SINGLE)); DocBuilder idb(s); - auto a1 = t.addAttribute("a1"); - auto a2 = t.addAttribute("a2"); - auto a3 = t.addAttribute("a3"); + auto a1 = t.addAttribute(a1_name); + auto a2 = t.addAttribute(a2_name); + auto a3 = t.addAttribute(a3_name); EXPECT_EQ(1u, a1->getNumDocs()); EXPECT_EQ(1u, a2->getNumDocs()); EXPECT_EQ(1u, a3->getNumDocs()); t.put(1, *idb.startDocument("id:ns:searchdocument::1"). - startAttributeField("a1").addInt(10).endField(). - startAttributeField("a2").addInt(15).endField(). - startAttributeField("a3").addInt(20).endField(). + startAttributeField(a1_name).addInt(10).endField(). + startAttributeField(a2_name).addInt(15).endField(). + startAttributeField(a3_name).addInt(20).endField(). endDocument(), 1); assertPutDone(*a1, 10); assertPutDone(*a2, 15); @@ -780,7 +787,7 @@ TEST_F(AttributeWriterTest, spreads_write_over_2_write_contexts) TEST_F(AttributeWriterTest, spreads_write_over_3_write_contexts) { setup(8); - putAttributes(*this, {0, 1, 3}); + putAttributes(*this, {4, 5, 6}); } struct MockPrepareResult : public PrepareResult { diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp index 2dc51d4577e..494b37297f4 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp @@ -120,6 +120,10 @@ TEST_F("require that all attributes can be retrieved", Fixture) std::vector<const IAttributeVector *> list; f.ctx->getAttributeList(list); EXPECT_EQUAL(2u, list.size()); + // Don't depend on internal (unspecified) ordering + std::sort(list.begin(), list.end(), [](auto* lhs, auto* rhs){ + return lhs->getName() < rhs->getName(); + }); EXPECT_EQUAL("bar", list[0]->getName()); EXPECT_EQUAL("foo", list[1]->getName()); } diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp index d92392ec8df..3155d7930e1 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp @@ -9,6 +9,7 @@ LOG_SETUP("imported_attributes_repo_test"); #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> #include <vespa/searchlib/attribute/reference_attribute.h> +#include <algorithm> using proton::ImportedAttributesRepo; using search::AttributeVector; @@ -74,6 +75,10 @@ TEST_F("require that all attributes can be retrieved", Fixture) std::vector<ImportedAttributeVector::SP> list; f.repo.getAll(list); EXPECT_EQUAL(2u, list.size()); + // Don't depend on internal (unspecified) ordering + std::sort(list.begin(), list.end(), [](auto& lhs, auto& rhs){ + return lhs->getName() < rhs->getName(); + }); EXPECT_EQUAL("bar", list[0]->getName()); EXPECT_EQUAL("foo", list[1]->getName()); } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index aebe5a61198..43f16e87986 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -24,7 +24,6 @@ #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/test/test.h> #include <vespa/searchcore/proton/test/thread_utils.h> -#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/test/directory_handler.h> diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index b275064c0f2..9122ea7850f 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -176,7 +176,7 @@ struct MyGidToLidChangeHandler : public MockGidToLidChangeHandler uint32_t _changes; std::map<document::GlobalId, uint32_t> _gidToLid; public: - MyGidToLidChangeHandler() + MyGidToLidChangeHandler() noexcept : MockGidToLidChangeHandler(), _changeGid(), _changeLid(std::numeric_limits<uint32_t>::max()), @@ -594,7 +594,8 @@ struct FixtureBase void putAndWait(const DocumentContext &docCtx) { FeedTokenContext token(_tracer); PutOperation op(docCtx.bid, docCtx.ts, docCtx.doc); - runInMaster([&] () { performPut(token.ft, op); }); + runInMaster([this, ft=std::move(token.ft), &op] () mutable { performPut(std::move(ft), op); }); + token.mt.await(); } void performUpdate(FeedToken token, UpdateOperation &op) { @@ -606,7 +607,8 @@ struct FixtureBase void updateAndWait(const DocumentContext &docCtx) { FeedTokenContext token(_tracer); UpdateOperation op(docCtx.bid, docCtx.ts, docCtx.upd); - runInMaster([&] () { performUpdate(token.ft, op); }); + runInMaster([this, ft=std::move(token.ft), &op] () mutable { performUpdate(std::move(ft), op); }); + token.mt.await(); } void performRemove(FeedToken token, RemoveOperation &op) { @@ -620,7 +622,8 @@ struct FixtureBase void removeAndWait(const DocumentContext &docCtx) { FeedTokenContext token(_tracer); RemoveOperationWithDocId op(docCtx.bid, docCtx.ts, docCtx.doc->getId()); - runInMaster([&] () { performRemove(token.ft, op); }); + runInMaster([this, ft=std::move(token.ft), &op] () mutable { performRemove(std::move(ft), op); }); + token.mt.await(); } void removeAndWait(const DocumentContext::List &docs) { @@ -647,7 +650,9 @@ struct FixtureBase } void performForceCommit() { getFeedView().forceCommit(serial); } - void forceCommitAndWait() { runInMaster([&]() { performForceCommit(); }); } + void forceCommitAndWait() { + runInMaster([&]() { performForceCommit(); }); + } bool assertTrace(const vespalib::string &exp) { return EXPECT_EQUAL(exp, _tracer._os.str()); diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index 04444647b5d..8890a6cfdda 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -212,11 +212,10 @@ MyDocumentStore::~MyDocumentStore() = default; struct MyDocumentRetriever : public DocumentRetrieverBaseForTest { std::shared_ptr<const DocumentTypeRepo> repo; const MyDocumentStore& store; - MyDocumentRetriever(std::shared_ptr<const DocumentTypeRepo> repo_in, const MyDocumentStore& store_in) + MyDocumentRetriever(std::shared_ptr<const DocumentTypeRepo> repo_in, const MyDocumentStore& store_in) noexcept : repo(std::move(repo_in)), store(store_in) - { - } + {} const document::DocumentTypeRepo& getDocumentTypeRepo() const override { return *repo; } void getBucketMetaData(const storage::spi::Bucket&, DocumentMetaData::Vector&) const override { abort(); } DocumentMetaData getDocumentMetaData(const DocumentId&) const override { abort(); } diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 70b2100521f..63c7a873eaf 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -147,7 +147,7 @@ struct MyDocumentRetriever : public DocumentRetrieverBaseForTest { MyDocumentSubDB &_subDB; - explicit MyDocumentRetriever(MyDocumentSubDB &subDB) + explicit MyDocumentRetriever(MyDocumentSubDB &subDB) noexcept : _subDB(subDB) { } diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index 8d1276497bd..e6e71d51e47 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -171,7 +171,7 @@ struct MoveOperationFeedView : public MyMinimalFeedView { struct MoveOperationCallback : public IDestructorCallback { int &outstandingMoveOps; - MoveOperationCallback(int &outstandingMoveOps_) : outstandingMoveOps(outstandingMoveOps_) { + explicit MoveOperationCallback(int &outstandingMoveOps_) noexcept : outstandingMoveOps(outstandingMoveOps_) { ++outstandingMoveOps; } ~MoveOperationCallback() override { @@ -194,7 +194,7 @@ struct FixtureBase { documentmetastore::LidReuseDelayerConfig lidReuseDelayerConfig; typename FeedViewType::UP feedview; - FixtureBase(SubDbType subDbType = SubDbType::READY) + explicit FixtureBase(SubDbType subDbType = SubDbType::READY) : removeCount(0), putCount(0), heartbeatCount(0), diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 9ca22bafe9a..60d96e37e15 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -2090,7 +2090,7 @@ public: size_t remove_batch_cnt; size_t remove_cnt; - MockOperationListener() + MockOperationListener() noexcept : remove_batch_cnt(0), remove_cnt(0) { diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index 08fa1b11229..38fca35ea87 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -361,14 +361,14 @@ public: public: typedef std::shared_ptr<SimpleStrategy> SP; - SimpleStrategy() {} + SimpleStrategy() noexcept : _targets() {} uint32_t indexOf(const IFlushTarget::SP &target) const { IFlushTarget *raw = target.get(); CachedFlushTarget *cached = dynamic_cast<CachedFlushTarget*>(raw); - if (cached != NULL) { + if (cached != nullptr) { raw = cached->getFlushTarget().get(); } WrappedFlushTarget *wrapped = dynamic_cast<WrappedFlushTarget *>(raw); diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 5f93f97f165..1b8be388c13 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -27,16 +27,16 @@ struct SimpleFlushTarget : public test::DummyFlushTarget const Type &type, SerialNum flushedSerial_, uint64_t approxDiskBytes_, - double replay_operation_cost_) + double replay_operation_cost_) noexcept : test::DummyFlushTarget(name, type, Component::OTHER), flushedSerial(flushedSerial_), approxDiskBytes(approxDiskBytes_), replay_operation_cost(replay_operation_cost_) {} - virtual SerialNum getFlushedSerialNum() const override { + SerialNum getFlushedSerialNum() const override { return flushedSerial; } - virtual uint64_t getApproxBytesToWriteToDisk() const override { + uint64_t getApproxBytesToWriteToDisk() const override { return approxDiskBytes; } double get_replay_operation_cost() const override { diff --git a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp index 964cd47afa5..6c502bccce1 100644 --- a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp +++ b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp @@ -18,26 +18,26 @@ class MyLidSpace : public search::common::ICompactableLidSpace bool _canShrink; size_t _canFree; public: - MyLidSpace() + MyLidSpace() noexcept : _canShrink(true), _canFree(16) { } - virtual ~MyLidSpace() override {} + ~MyLidSpace() override = default; - virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { + void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; } - virtual bool canShrinkLidSpace() const override { + bool canShrinkLidSpace() const override { return _canShrink; } - virtual size_t getEstimatedShrinkLidSpaceGain() const override { + size_t getEstimatedShrinkLidSpaceGain() const override { return _canShrink ? _canFree : 0; } - virtual void shrinkLidSpace() override { + void shrinkLidSpace() override { if (_canShrink) { _canFree = 0; _canShrink = false; @@ -62,9 +62,11 @@ struct Fixture { } - ~Fixture() { } + ~Fixture(); }; +Fixture::~Fixture() = default; + TEST_F("require that flush target returns estimated memory gain", Fixture) { auto memoryGain = f._ft->getApproxMemoryGain(); diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 22dc19167f6..30f83273bd7 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -284,7 +284,7 @@ struct MyWorld { struct MySearchHandler : ISearchHandler { Matcher::SP _matcher; - MySearchHandler(Matcher::SP matcher) : _matcher(matcher) {} + MySearchHandler(Matcher::SP matcher) noexcept : _matcher(std::move(matcher)) {} DocsumReply::UP getDocsums(const DocsumRequest &) override { return DocsumReply::UP(); diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 1e64a8f4ecb..93fe2f0ae24 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -14,6 +14,7 @@ #include <vespa/fileacquirer/config-filedistributorrpc.h> #include <vespa/vespalib/util/varholder.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/config/common/configcontext.h> #include <vespa/config-bucketspaces.h> #include <vespa/config-attributes.h> #include <vespa/config-imported-fields.h> @@ -73,7 +74,7 @@ struct ConfigTestFixture { bucketspacesBuilder(), dbConfig(), set(), - context(new ConfigContext(set)), + context(std::make_shared<ConfigContext>(set)), idcounter(-1) { set.addBuilder(configId, &protonBuilder); diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp index 48b2a269f6b..bacdd85dcc5 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp @@ -61,7 +61,7 @@ struct MyDocumentDBReference : public MockDocumentDBReference { std::shared_ptr<MockGidToLidChangeHandler> _gidToLidChangeHandler; MyDocumentDBReference(MyGidToLidMapperFactory::SP factory_, - std::shared_ptr<MockGidToLidChangeHandler> gidToLidChangeHandler) + std::shared_ptr<MockGidToLidChangeHandler> gidToLidChangeHandler) noexcept : factory(std::move(factory_)), _gidToLidChangeHandler(std::move(gidToLidChangeHandler)) { diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp index 920472c6a01..a10d48ee7fe 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -39,7 +39,7 @@ class ListenerStats { uint32_t _destroyedListeners; public: - ListenerStats() + ListenerStats() noexcept : _lock(), _putChanges(0u), _removeChanges(0u), diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp index 0053a03105f..242ad91271e 100644 --- a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp @@ -32,7 +32,7 @@ struct MyTask : public IReprocessingTask double initProgress, double middleProgress, double finalProgress, - double weight) + double weight) noexcept : _runner(runner), _initProgress(initProgress), _middleProgress(middleProgress), @@ -42,9 +42,7 @@ struct MyTask : public IReprocessingTask { } - virtual void - run() override - { + void run() override { ASSERT_EQUAL(_initProgress, _runner.getProgress()); _myProgress = 0.5; ASSERT_EQUAL(_middleProgress, _runner.getProgress()); @@ -52,9 +50,7 @@ struct MyTask : public IReprocessingTask ASSERT_EQUAL(_finalProgress, _runner.getProgress()); } - virtual Progress - getProgress() const override - { + Progress getProgress() const override { return Progress(_myProgress, _weight); } @@ -65,11 +61,7 @@ struct MyTask : public IReprocessingTask double finalProgress, double weight) { - return std::make_shared<MyTask>(runner, - initProgress, - middleProgress, - finalProgress, - weight); + return std::make_shared<MyTask>(runner, initProgress, middleProgress, finalProgress, weight); } }; @@ -77,16 +69,8 @@ TEST_F("require that progress is calculated when tasks are executed", Fixture) { TaskList tasks; EXPECT_EQUAL(0.0, f._runner.getProgress()); - tasks.push_back(MyTask::create(f._runner, - 0.0, - 0.1, - 0.2, - 1.0)); - tasks.push_back(MyTask::create(f._runner, - 0.2, - 0.6, - 1.0, - 4.0)); + tasks.push_back(MyTask::create(f._runner, 0.0, 0.1, 0.2, 1.0)); + tasks.push_back(MyTask::create(f._runner, 0.2, 0.6, 1.0, 4.0)); f._runner.addTasks(tasks); tasks.clear(); EXPECT_EQUAL(0.0, f._runner.getProgress()); @@ -99,11 +83,7 @@ TEST_F("require that runner can be reset", Fixture) { TaskList tasks; EXPECT_EQUAL(0.0, f._runner.getProgress()); - tasks.push_back(MyTask::create(f._runner, - 0.0, - 0.5, - 1.0, - 1.0)); + tasks.push_back(MyTask::create(f._runner, 0.0, 0.5, 1.0, 1.0)); f._runner.addTasks(tasks); tasks.clear(); EXPECT_EQUAL(0.0, f._runner.getProgress()); @@ -111,21 +91,13 @@ TEST_F("require that runner can be reset", Fixture) EXPECT_EQUAL(1.0, f._runner.getProgress()); f._runner.reset(); EXPECT_EQUAL(0.0, f._runner.getProgress()); - tasks.push_back(MyTask::create(f._runner, - 0.0, - 0.5, - 1.0, - 1.0)); + tasks.push_back(MyTask::create(f._runner, 0.0, 0.5, 1.0, 1.0)); f._runner.addTasks(tasks); tasks.clear(); EXPECT_EQUAL(0.0, f._runner.getProgress()); f._runner.reset(); EXPECT_EQUAL(0.0, f._runner.getProgress()); - tasks.push_back(MyTask::create(f._runner, - 0.0, - 0.5, - 1.0, - 4.0)); + tasks.push_back(MyTask::create(f._runner, 0.0, 0.5, 1.0, 4.0)); f._runner.addTasks(tasks); tasks.clear(); EXPECT_EQUAL(0.0, f._runner.getProgress()); diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index 42d88328d14..59e504de6ce 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -124,7 +124,7 @@ TEST_F("require that replay progress is tracked", Fixture) { RemoveOperationContext opCtx(10); TlsReplayProgress progress("test", 5, 15); - PacketWrapper::SP wrap(new PacketWrapper(*opCtx.packet, &progress)); + auto wrap = std::make_shared<PacketWrapper>(*opCtx.packet, &progress); ForegroundThreadExecutor executor; f.state.receive(wrap, executor); diff --git a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp index 7f650847183..305f5d8c9ba 100644 --- a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp +++ b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp @@ -22,19 +22,13 @@ typedef IFlushTarget::DiskGain DiskGain; class MyFlushHandler : public IFlushHandler { public: - MyFlushHandler(const vespalib::string &name) : IFlushHandler(name) {} - // Implements IFlushHandler - virtual std::vector<IFlushTarget::SP> getFlushTargets() override { + MyFlushHandler(const vespalib::string &name) noexcept : IFlushHandler(name) {} + std::vector<IFlushTarget::SP> getFlushTargets() override { return std::vector<IFlushTarget::SP>(); } - virtual SerialNum getCurrentSerialNumber() const override { return 0; } - virtual void flushDone(SerialNum oldestSerial) override { (void) oldestSerial; } - - virtual void - syncTls(search::SerialNum syncTo) override - { - (void) syncTo; - } + SerialNum getCurrentSerialNumber() const override { return 0; } + void flushDone(SerialNum oldestSerial) override { (void) oldestSerial; } + void syncTls(search::SerialNum syncTo) override {(void) syncTo;} }; class MyFlushTarget : public test::DummyFlushTarget { @@ -47,7 +41,7 @@ private: public: MyFlushTarget(const vespalib::string &name, MemoryGain memoryGain, DiskGain diskGain, SerialNum flushedSerial, - system_time lastFlushTime, bool urgentFlush) : + system_time lastFlushTime, bool urgentFlush) noexcept : test::DummyFlushTarget(name), _memoryGain(memoryGain), _diskGain(diskGain), @@ -56,12 +50,11 @@ public: _urgentFlush(urgentFlush) { } - // Implements IFlushTarget - virtual MemoryGain getApproxMemoryGain() const override { return _memoryGain; } - virtual DiskGain getApproxDiskGain() const override { return _diskGain; } - virtual SerialNum getFlushedSerialNum() const override { return _flushedSerial; } - virtual system_time getLastFlushTime() const override { return _lastFlushTime; } - virtual bool needUrgentFlush() const override { return _urgentFlush; } + MemoryGain getApproxMemoryGain() const override { return _memoryGain; } + DiskGain getApproxDiskGain() const override { return _diskGain; } + SerialNum getFlushedSerialNum() const override { return _flushedSerial; } + system_time getLastFlushTime() const override { return _lastFlushTime; } + bool needUrgentFlush() const override { return _urgentFlush; } }; struct StringList : public std::vector<vespalib::string> { diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index fd60781d868..bcc35204efd 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -135,7 +135,7 @@ indexing.tasklimit int default=1000 restart ## nonzero. The number is divided by the number of indexing threads, ## i.e. when indexing.threads is 4 and indexing.semiunboundtasklimit ## is 40000 then effective task limit is 10000. -indexing.semiunboundtasklimit int default = 40000 restart +indexing.semiunboundtasklimit int default = 1000 restart ## Kind of watermark for when to activate extra manpower ## Utilized if optimize is set to either THROUGHPUT or ADAPTIVE diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp index 5208195a968..33a5776cb8a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp @@ -18,7 +18,7 @@ class PopulateDoneContext : public IDestructorCallback { std::shared_ptr<document::Document> _doc; public: - PopulateDoneContext(std::shared_ptr<document::Document> doc) + PopulateDoneContext(std::shared_ptr<document::Document> doc) noexcept : _doc(std::move(doc)) { } diff --git a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp index 21b27ef6ffd..4d76018d66b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp @@ -7,12 +7,12 @@ namespace proton { -DocTypeName::DocTypeName(const search::engine::Request &request) +DocTypeName::DocTypeName(const search::engine::Request &request) noexcept : _name(request.propertiesMap.matchProperties().lookup("documentdb", "searchdoctype").get("")) {} -DocTypeName::DocTypeName(const document::DocumentType &docType) +DocTypeName::DocTypeName(const document::DocumentType &docType) noexcept : _name(docType.getName()) {} diff --git a/searchcore/src/vespa/searchcore/proton/common/doctypename.h b/searchcore/src/vespa/searchcore/proton/common/doctypename.h index 661d54ab8d4..f0fa3c72ee9 100644 --- a/searchcore/src/vespa/searchcore/proton/common/doctypename.h +++ b/searchcore/src/vespa/searchcore/proton/common/doctypename.h @@ -13,9 +13,9 @@ class DocTypeName vespalib::string _name; public: - explicit DocTypeName(const vespalib::string &name) : _name(name) { } - explicit DocTypeName(const search::engine::Request &request); - explicit DocTypeName(const document::DocumentType &docType); + explicit DocTypeName(const vespalib::string &name) noexcept : _name(name) { } + explicit DocTypeName(const search::engine::Request &request) noexcept; + explicit DocTypeName(const document::DocumentType &docType) noexcept; const vespalib::string & getName() const { return _name; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/gid_compare.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/gid_compare.h index dd32abcab6e..c16dd8a7292 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/gid_compare.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/gid_compare.h @@ -14,7 +14,7 @@ class IGidCompare public: typedef std::shared_ptr<IGidCompare> SP; - virtual ~IGidCompare() {} + virtual ~IGidCompare() = default; virtual bool operator()(const document::GlobalId &lhs, const document::GlobalId &rhs) const = 0; @@ -30,14 +30,13 @@ private: document::GlobalId::BucketOrderCmp _comp; public: - DefaultGidCompare() + DefaultGidCompare() noexcept : IGidCompare(), _comp() { } - virtual bool operator()(const document::GlobalId &lhs, - const document::GlobalId &rhs) const override { + bool operator()(const document::GlobalId &lhs, const document::GlobalId &rhs) const override { return _comp(lhs, rhs); } }; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h index 914de9df30c..ca23f27ed49 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h @@ -28,7 +28,7 @@ public: * * @param name The unique name of this handler. */ - IFlushHandler(const vespalib::string &name) + IFlushHandler(const vespalib::string &name) noexcept : _name(name) { } diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h index f28c46334bc..bd0d82b0b2c 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h +++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h @@ -28,7 +28,7 @@ class TaskRunner { using SP = std::shared_ptr<Context>; Context(InitializerTask::SP rootTask, vespalib::Executor &contextExecutor, - vespalib::Executor::Task::UP doneTask) + vespalib::Executor::Task::UP doneTask) noexcept : _rootTask(rootTask), _contextExecutor(contextExecutor), _doneTask(std::move(doneTask)) diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h index 5bcc8dafea2..ac3c6ab5e0f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h +++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/dual_merge_director.h> #include <vespa/searchlib/common/rankedhit.h> #include <vector> +#include <cassert> namespace proton::matching { @@ -28,7 +29,7 @@ private: public: PartialResult(size_t maxSize_in, bool hasSortData_in); - ~PartialResult(); + ~PartialResult() override; size_t size() const { return _hits.size(); } size_t maxSize() const { return _maxSize; } size_t totalHits() const { return _totalHits; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h index 96c83270735..0cff5583bbd 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h +++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h @@ -37,7 +37,7 @@ public: vespalib::steady_time doom; SearchSessionInfo(const vespalib::string &id_in, vespalib::steady_time created_in, - vespalib::steady_time doom_in) + vespalib::steady_time doom_in) noexcept : id(id_in), created(created_in), doom(doom_in) {} }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index c8b701e82f8..e842e3cdc41 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -9,6 +9,7 @@ #include <vespa/config-rank-profiles.h> #include <vespa/config-summarymap.h> #include <vespa/config/file_acquirer/file_acquirer.h> +#include <vespa/config/common/configcontext.h> #include <vespa/config/helper/legacy.h> #include <vespa/config-attributes.h> #include <vespa/config-indexschema.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 174ba090842..a52be734c13 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -111,7 +111,7 @@ TlsMgrWriter::sync(SerialNum syncTo) class OnCommitDone : public search::IDestructorCallback { public: - OnCommitDone(Executor & executor, std::unique_ptr<Executor::Task> task) + OnCommitDone(Executor & executor, std::unique_ptr<Executor::Task> task) noexcept : _executor(executor), _task(std::move(task)) {} diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.h b/searchcore/src/vespa/searchcore/proton/server/feedstates.h index 0c2e9109cce..3d24f4068a2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.h @@ -24,7 +24,7 @@ class InitState : public FeedState { vespalib::string _doc_type_name; public: - InitState(const vespalib::string &name) + InitState(const vespalib::string &name) noexcept : FeedState(INIT), _doc_type_name(name) { @@ -73,7 +73,7 @@ class NormalState : public FeedState { FeedHandler &_handler; public: - NormalState(FeedHandler &handler) + NormalState(FeedHandler &handler) noexcept : FeedState(NORMAL), _handler(handler) { } diff --git a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp index e535b05393c..c84abb373f8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp @@ -10,7 +10,7 @@ using BlockedReason = IBlockableMaintenanceJob::BlockedReason; struct MoveOperationLimiter::Callback : public search::IDestructorCallback { MoveOperationLimiter::SP _limiter; - Callback(MoveOperationLimiter::SP limiter) : _limiter(std::move(limiter)) {} + Callback(MoveOperationLimiter::SP limiter) noexcept : _limiter(std::move(limiter)) {} virtual ~Callback() { _limiter->endOperation(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h b/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h index c36652ec847..6f132604662 100644 --- a/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h @@ -20,8 +20,7 @@ struct PacketWrapper { search::transactionlog::client::RPC::Result result; vespalib::Gate gate; - PacketWrapper(const search::transactionlog::Packet &p, - TlsReplayProgress *progress_) + PacketWrapper(const search::transactionlog::Packet &p, TlsReplayProgress *progress_) noexcept : packet(p), progress(progress_), result(search::transactionlog::client::RPC::ERROR), diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index c724e1065e9..428aba1a6b5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -19,7 +19,7 @@ namespace { struct Pair { string key; string value; - Pair(const string &k, const string &v) + Pair(const string &k, const string &v) noexcept : key(k), value(v) { @@ -200,7 +200,7 @@ RPCHooksBase::RPCHooksBase(Params ¶ms) _proton.get_search_server(), _proton.get_docsum_server(), _proton.get_monitor_server(), *_orb)), - _regAPI(*_orb, params.slobrok_config), + _regAPI(*_orb, slobrok::ConfiguratorFactory(params.slobrok_config)), _stateLock(), _stateCond(), _executor(48, 128 * 1024) diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index cf9f27a94af..3b268835331 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -60,7 +60,7 @@ ThreadingServiceConfig::make(const ProtonConfig &cfg, double concurrency, const { uint32_t indexingThreads = calculateIndexingThreads(cfg.indexing, concurrency, cpuInfo); return ThreadingServiceConfig(indexingThreads, cfg.indexing.tasklimit, - (cfg.indexing.semiunboundtasklimit / indexingThreads), + std::max(uint32_t(cfg.indexing.tasklimit), (cfg.indexing.semiunboundtasklimit / indexingThreads)), selectOptimization(cfg.indexing.optimize), cfg.indexing.kindOfWatermark, vespalib::from_s(cfg.indexing.reactiontime)); diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h index 66cc5ed0fea..a6ff6d86d0d 100644 --- a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h @@ -15,38 +15,36 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore DocId _compactLidSpaceLidLimit; uint32_t _holdUnblockShrinkLidSpaceCnt; - DocumentMetaStoreObserver(IDocumentMetaStore &store) + DocumentMetaStoreObserver(IDocumentMetaStore &store) noexcept : _store(store), _removeCompleteCnt(0), _removeCompleteLid(0), _compactLidSpaceLidLimit(0), _holdUnblockShrinkLidSpaceCnt(0) - { - } + {} /** * Implements search::IDocumentMetaStore **/ - virtual bool getGid(DocId lid, GlobalId &gid) const override { + bool getGid(DocId lid, GlobalId &gid) const override { return _store.getGid(lid, gid); } - virtual bool getGidEvenIfMoved(DocId lid, GlobalId &gid) const override { + bool getGidEvenIfMoved(DocId lid, GlobalId &gid) const override { return _store.getGidEvenIfMoved(lid, gid); } - virtual bool getLid(const GlobalId &gid, DocId &lid) const override { + bool getLid(const GlobalId &gid, DocId &lid) const override { return _store.getLid(gid, lid); } - virtual search::DocumentMetaData getMetaData(const GlobalId &gid) const override { + search::DocumentMetaData getMetaData(const GlobalId &gid) const override { return _store.getMetaData(gid); } - virtual void getMetaData(const BucketId &bucketId, - search::DocumentMetaData::Vector &result) const override { + void getMetaData(const BucketId &bucketId, search::DocumentMetaData::Vector &result) const override { _store.getMetaData(bucketId, result); } - virtual search::LidUsageStats getLidUsageStats() const override { + search::LidUsageStats getLidUsageStats() const override { return _store.getLidUsageStats(); } - virtual search::queryeval::Blueprint::UP createWhiteListBlueprint() const override { + search::queryeval::Blueprint::UP createWhiteListBlueprint() const override { return _store.createWhiteListBlueprint(); } uint64_t getCurrentGeneration() const override { @@ -57,40 +55,39 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore /** * Implements documentmetastore::IStore. */ - virtual Result inspectExisting(const GlobalId &gid, uint64_t prepare_serial_num) override { + Result inspectExisting(const GlobalId &gid, uint64_t prepare_serial_num) override { return _store.inspectExisting(gid, prepare_serial_num); } - virtual Result inspect(const GlobalId &gid, uint64_t prepare_serial_num) override { + Result inspect(const GlobalId &gid, uint64_t prepare_serial_num) override { return _store.inspect(gid, prepare_serial_num); } - virtual Result put(const GlobalId &gid, - const BucketId &bucketId, - const Timestamp ×tamp, - uint32_t docSize, - DocId lid, - uint64_t prepare_serial_num) override { + Result put(const GlobalId &gid, + const BucketId &bucketId, + const Timestamp ×tamp, + uint32_t docSize, + DocId lid, + uint64_t prepare_serial_num) override + { return _store.put(gid, bucketId, timestamp, docSize, lid, prepare_serial_num); } - virtual bool updateMetaData(DocId lid, - const BucketId &bucketId, - const Timestamp ×tamp) override { + bool updateMetaData(DocId lid, const BucketId &bucketId, const Timestamp ×tamp) override { return _store.updateMetaData(lid, bucketId, timestamp); } - virtual bool remove(DocId lid, uint64_t prepare_serial_num) override { + bool remove(DocId lid, uint64_t prepare_serial_num) override { return _store.remove(lid, prepare_serial_num); } - virtual void removeComplete(DocId lid) override { + void removeComplete(DocId lid) override { ++_removeCompleteCnt; _removeCompleteLid = lid; _store.removeComplete(lid); } - virtual void move(DocId fromLid, DocId toLid, uint64_t prepare_serial_num) override { + void move(DocId fromLid, DocId toLid, uint64_t prepare_serial_num) override { _store.move(fromLid, toLid, prepare_serial_num); } - virtual bool validLid(DocId lid) const override { + bool validLid(DocId lid) const override { return _store.validLid(lid); } - virtual void removeBatch(const std::vector<DocId> &lidsToRemove, + void removeBatch(const std::vector<DocId> &lidsToRemove, const DocId docIdLimit) override { _store.removeBatch(lidsToRemove, docIdLimit); } diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_handler.h b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_handler.h index 53fa8a9c3fb..af2f4c63443 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_handler.h @@ -3,37 +3,32 @@ #include <vespa/searchcore/proton/flushengine/iflushhandler.h> -namespace proton { - -namespace test { +namespace proton::test { /** * Default implementation used for testing. */ struct DummyFlushHandler : public IFlushHandler { - DummyFlushHandler(const vespalib::string &name) + DummyFlushHandler(const vespalib::string &name) noexcept : IFlushHandler(name) {} - // Implements IFlushHandler - virtual std::vector<IFlushTarget::SP> getFlushTargets() override { + std::vector<IFlushTarget::SP> getFlushTargets() override { return std::vector<IFlushTarget::SP>(); } - virtual SerialNum getCurrentSerialNumber() const override { + SerialNum getCurrentSerialNumber() const override { return 0; } - virtual void flushDone(SerialNum oldestSerial) override { + void flushDone(SerialNum oldestSerial) override { (void) oldestSerial; } - virtual void syncTls(SerialNum syncTo) override { + void syncTls(SerialNum syncTo) override { (void) syncTo; } }; -} // namespace test - -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h index 8e291126430..dd3cd49df2f 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h @@ -3,39 +3,33 @@ #include <vespa/searchcorespi/flush/iflushtarget.h> -namespace proton { - -namespace test { +namespace proton::test { struct DummyFlushTarget : public searchcorespi::IFlushTarget { - DummyFlushTarget(const vespalib::string &name) + DummyFlushTarget(const vespalib::string &name) noexcept : searchcorespi::IFlushTarget(name) {} DummyFlushTarget(const vespalib::string &name, const Type &type, - const Component &component) + const Component &component) noexcept : searchcorespi::IFlushTarget(name, type, component) {} - // Implements searchcorespi::IFlushTarget - virtual MemoryGain getApproxMemoryGain() const override { return MemoryGain(0, 0); } - virtual DiskGain getApproxDiskGain() const override { return DiskGain(0, 0); } - virtual SerialNum getFlushedSerialNum() const override { return 0; } - virtual Time getLastFlushTime() const override { return Time(); } - virtual bool needUrgentFlush() const override { return false; } - virtual searchcorespi::FlushTask::UP initFlush(SerialNum) override { + MemoryGain getApproxMemoryGain() const override { return MemoryGain(0, 0); } + DiskGain getApproxDiskGain() const override { return DiskGain(0, 0); } + SerialNum getFlushedSerialNum() const override { return 0; } + Time getLastFlushTime() const override { return Time(); } + bool needUrgentFlush() const override { return false; } + searchcorespi::FlushTask::UP initFlush(SerialNum) override { return searchcorespi::FlushTask::UP(); } - virtual searchcorespi::FlushStats getLastFlushStats() const override { + searchcorespi::FlushStats getLastFlushStats() const override { return searchcorespi::FlushStats(); } - virtual uint64_t getApproxBytesToWriteToDisk() const override { + uint64_t getApproxBytesToWriteToDisk() const override { return 0; } }; -} // namespace test - -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h index e808f721e83..f9531486e9b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h @@ -23,7 +23,7 @@ private: std::vector<std::unique_ptr<IGidToLidChangeListener>> _listeners; public: - MockGidToLidChangeHandler() + MockGidToLidChangeHandler() noexcept : IGidToLidChangeHandler(), _adds(), _removes(), @@ -31,7 +31,7 @@ public: { } - ~MockGidToLidChangeHandler() override { } + ~MockGidToLidChangeHandler() override = default; void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override { _adds.emplace_back(listener->getDocTypeName(), listener->getName()); diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h index 4dac300d614..340d3b08441 100644 --- a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h +++ b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h @@ -4,9 +4,7 @@ #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/vespalib/util/closuretask.h> -namespace proton { - -namespace test { +namespace proton::test { template <typename FunctionType> void @@ -20,15 +18,11 @@ runFunction(FunctionType *func) */ template <typename FunctionType> void -runInMaster(searchcorespi::index::IThreadingService &writeService, - FunctionType func) +runInMaster(searchcorespi::index::IThreadingService &writeService, FunctionType func) { writeService.master().execute(vespalib::makeTask (vespalib::makeClosure(&runFunction<FunctionType>, &func))); writeService.sync(); } -} // namespace test - -} // namespace proton - +} diff --git a/searchcorespi/CMakeLists.txt b/searchcorespi/CMakeLists.txt index 32469886bf8..2faa2be3b82 100644 --- a/searchcorespi/CMakeLists.txt +++ b/searchcorespi/CMakeLists.txt @@ -18,8 +18,4 @@ vespa_define_module( src/vespa/searchcorespi src/vespa/searchcorespi/flush src/vespa/searchcorespi/index - src/vespa/searchcorespi/plugin - - TESTS - src/tests/plugin ) diff --git a/searchcorespi/src/tests/plugin/.gitignore b/searchcorespi/src/tests/plugin/.gitignore deleted file mode 100644 index e49000038ad..00000000000 --- a/searchcorespi/src/tests/plugin/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -Makefile -.depend -*_test -searchcorespi_factoryregistry_test_app -searchcorespi_plugin_test_app diff --git a/searchcorespi/src/tests/plugin/CMakeLists.txt b/searchcorespi/src/tests/plugin/CMakeLists.txt deleted file mode 100644 index da785b09b6a..00000000000 --- a/searchcorespi/src/tests/plugin/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcorespi_tplugin - SOURCES - plugin.cpp - DEPENDS - searchcorespi -) -vespa_add_library(searchcorespi_illegal-plugin - SOURCES - empty.cpp - DEPENDS - searchcorespi -) -vespa_add_executable(searchcorespi_plugin_test_app TEST - SOURCES - plugin_test.cpp - DEPENDS - searchcorespi -) -vespa_add_test( - NAME searchcorespi_plugin_test_app - COMMAND searchcorespi_plugin_test_app - ENVIRONMENT "LD_LIBRARY_PATH=." - DEPENDS searchcorespi_tplugin searchcorespi_illegal-plugin -) -vespa_add_executable(searchcorespi_factoryregistry_test_app TEST - SOURCES - factoryregistry_test.cpp - DEPENDS - searchcorespi -) -vespa_add_test(NAME searchcorespi_factoryregistry_test_app COMMAND searchcorespi_factoryregistry_test_app) diff --git a/searchcorespi/src/tests/plugin/empty.cpp b/searchcorespi/src/tests/plugin/empty.cpp deleted file mode 100644 index c7c47fc98c5..00000000000 --- a/searchcorespi/src/tests/plugin/empty.cpp +++ /dev/null @@ -1 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. diff --git a/searchcorespi/src/tests/plugin/factoryregistry_test.cpp b/searchcorespi/src/tests/plugin/factoryregistry_test.cpp deleted file mode 100644 index 10a1c95f191..00000000000 --- a/searchcorespi/src/tests/plugin/factoryregistry_test.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/searchcorespi/plugin/factoryregistry.h> -#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/exceptions.h> - -using vespalib::string; -using namespace searchcorespi; - -namespace { - -struct MyFactory : IIndexManagerFactory { - - virtual IIndexManager::UP createIndexManager(const IndexManagerConfig &, - const index::IndexMaintainerConfig &, - const index::IndexMaintainerContext &) override { - return IIndexManager::UP(); - } - virtual config::ConfigKeySet getConfigKeys( - const string &, - const search::index::Schema &, - const config::ConfigInstance &) override { - return config::ConfigKeySet(); - } -}; - -const string name = "factory"; - -TEST("require that factories can be added and removed") { - FactoryRegistry registry; - EXPECT_FALSE(registry.isRegistered(name)); - registry.add(name, IIndexManagerFactory::SP(new MyFactory)); - EXPECT_TRUE(registry.get(name).get()); - EXPECT_TRUE(registry.isRegistered(name)); - registry.remove(name); - EXPECT_EXCEPTION(registry.get(name), vespalib::IllegalArgumentException, - "No factory is registered with the name"); -} - -TEST("require that two factories with the same name cannot be added") { - FactoryRegistry registry; - registry.add(name, IIndexManagerFactory::SP(new MyFactory)); - EXPECT_EXCEPTION( - registry.add(name, IIndexManagerFactory::SP(new MyFactory)), - vespalib::IllegalArgumentException, - "A factory is already registered with the same name"); -} - -TEST("require that a non-existent factory cannot be removed") { - FactoryRegistry registry; - EXPECT_EXCEPTION(registry.remove(name), vespalib::IllegalArgumentException, - "No factory is registered with the name"); -} - -} // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcorespi/src/tests/plugin/plugin.cpp b/searchcorespi/src/tests/plugin/plugin.cpp deleted file mode 100644 index d32b02f45fd..00000000000 --- a/searchcorespi/src/tests/plugin/plugin.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> - -using namespace search; -using namespace search::index; -using namespace vespalib; -using namespace config; - -namespace searchcorespi { -class IndexManager : public searchcorespi::IIndexManager -{ -public: - - typedef search::SerialNum SerialNum; - typedef search::index::Schema Schema; - typedef document::Document Document; - using OnWriteDoneType = - const std::shared_ptr<search::IDestructorCallback> &; - virtual void putDocument(uint32_t, const Document &, SerialNum) override { } - virtual void removeDocument(uint32_t, SerialNum) override { } - virtual void commit(SerialNum, OnWriteDoneType) override { } - virtual void heartBeat(SerialNum ) override {} - void compactLidSpace(uint32_t, SerialNum) override {} - virtual SerialNum getCurrentSerialNum() const override { return 0; } - virtual SerialNum getFlushedSerialNum() const override { return 0; } - virtual IndexSearchable::SP getSearchable() const override { - IndexSearchable::SP s; - return s; - } - virtual SearchableStats getSearchableStats() const override { - SearchableStats s; - return s; - } - virtual searchcorespi::IFlushTarget::List getFlushTargets() override { - searchcorespi::IFlushTarget::List l; - return l; - } - virtual void setSchema(const Schema &, SerialNum) override { } - virtual void setMaxFlushed(uint32_t) override { } -}; - -class IndexManagerFactory : public searchcorespi::IIndexManagerFactory -{ -public: - virtual IIndexManager::UP createIndexManager(const IndexManagerConfig &managerCfg, - const index::IndexMaintainerConfig &maintainerConfig, - const index::IndexMaintainerContext &maintainerContext) override; - - virtual ConfigKeySet getConfigKeys(const string &configId, - const Schema &schema, - const ConfigInstance &rootConfig) override; -}; - -IIndexManager::UP -IndexManagerFactory::createIndexManager(const IndexManagerConfig &, - const index::IndexMaintainerConfig &, - const index::IndexMaintainerContext &) -{ - return IIndexManager::UP(new IndexManager()); -} - -ConfigKeySet -IndexManagerFactory::getConfigKeys(const string &, - const Schema &, - const ConfigInstance &) -{ - ConfigKeySet keys; - return keys; -} - -} - -searchcorespi::IIndexManagerFactory * -createIndexManagerFactory() -{ - return new searchcorespi::IndexManagerFactory(); -} - diff --git a/searchcorespi/src/tests/plugin/plugin_test.cpp b/searchcorespi/src/tests/plugin/plugin_test.cpp deleted file mode 100644 index 733c2834c24..00000000000 --- a/searchcorespi/src/tests/plugin/plugin_test.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchcorespi/plugin/factoryloader.h> -#include <vespa/vespalib/util/exceptions.h> - -using namespace searchcorespi; - -namespace { -TEST("require that plugins can be loaded.") { - FactoryLoader fl; - IIndexManagerFactory::UP f = fl.create("searchcorespi_tplugin"); - ASSERT_TRUE(f.get()); -} - -TEST("require that non-existent plugin causes failure") { - FactoryLoader fl; -#ifdef __APPLE__ - EXPECT_EXCEPTION(fl.create("no-such-plugin"), - vespalib::IllegalArgumentException, - "image not found"); -#else - EXPECT_EXCEPTION(fl.create("no-such-plugin"), - vespalib::IllegalArgumentException, - "cannot open shared object file"); -#endif -} - -TEST("require that missing factory function causes failure") { - FactoryLoader fl; - EXPECT_EXCEPTION(fl.create("searchcorespi_illegal-plugin"), - vespalib::IllegalArgumentException, - "Failed locating symbol 'createIndexManagerFactory'"); -} -} // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcorespi/src/vespa/searchcorespi/CMakeLists.txt b/searchcorespi/src/vespa/searchcorespi/CMakeLists.txt index 3cbe1265136..bee83a7f936 100644 --- a/searchcorespi/src/vespa/searchcorespi/CMakeLists.txt +++ b/searchcorespi/src/vespa/searchcorespi/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(searchcorespi SOURCES $<TARGET_OBJECTS:searchcorespi_flush> $<TARGET_OBJECTS:searchcorespi_index> - $<TARGET_OBJECTS:searchcorespi_plugin> INSTALL lib64 DEPENDS ) diff --git a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h index 10ed19a7244..3a67333c9d5 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h +++ b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h @@ -45,8 +45,8 @@ public: template<typename T> class Gain { public: - Gain() : _before(0), _after(0) { } - Gain(T before, T after) : _before(before), _after(after) { } + Gain() noexcept : _before(0), _after(0) { } + Gain(T before, T after) noexcept : _before(before), _after(after) { } T getBefore() const { return _before; } T getAfter() const { return _after; } T gain() const { return _before - _after; } @@ -74,7 +74,7 @@ public: * * @param name The handler-wide unique name of this target. */ - IFlushTarget(const vespalib::string &name) + IFlushTarget(const vespalib::string &name) noexcept : _name(name), _type(Type::OTHER), _component(Component::OTHER) @@ -89,7 +89,7 @@ public: */ IFlushTarget(const vespalib::string &name, const Type &type, - const Component &component) + const Component &component) noexcept : _name(name), _type(type), _component(component) @@ -98,7 +98,7 @@ public: /** * Virtual destructor required for inheritance. */ - virtual ~IFlushTarget() { } + virtual ~IFlushTarget() = default; /** * Returns the handler-wide unique name of this target. diff --git a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp index b8572541f36..bec6f02324a 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp @@ -1,27 +1,29 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "activediskindexes.h" +#include <cassert> -using std::set; using vespalib::string; -using vespalib::LockGuard; namespace searchcorespi::index { +ActiveDiskIndexes::ActiveDiskIndexes() = default; +ActiveDiskIndexes::~ActiveDiskIndexes() = default; + void ActiveDiskIndexes::setActive(const string &index) { - LockGuard lock(_lock); + std::lock_guard lock(_lock); _active.insert(index); } void ActiveDiskIndexes::notActive(const string & index) { - LockGuard lock(_lock); - set<string>::iterator it = _active.find(index); + std::lock_guard lock(_lock); + auto it = _active.find(index); assert(it != _active.end()); _active.erase(it); } bool ActiveDiskIndexes::isActive(const string &index) const { - LockGuard lock(_lock); + std::lock_guard lock(_lock); return _active.find(index) != _active.end(); } diff --git a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h index dff25906559..c1c0e4e4d88 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h +++ b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h @@ -3,11 +3,10 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/sync.h> #include <set> +#include <mutex> -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { /** * Class used to keep track of the set of active disk indexes in an index maintainer. @@ -15,16 +14,17 @@ namespace index { */ class ActiveDiskIndexes { std::multiset<vespalib::string> _active; - vespalib::Lock _lock; + mutable std::mutex _lock; public: - typedef std::shared_ptr<ActiveDiskIndexes> SP; - + using SP = std::shared_ptr<ActiveDiskIndexes>; + ActiveDiskIndexes(); + ~ActiveDiskIndexes(); + ActiveDiskIndexes(const ActiveDiskIndexes &) = delete; + ActiveDiskIndexes & operator = (const ActiveDiskIndexes &) = delete; void setActive(const vespalib::string & index); void notActive(const vespalib::string & index); bool isActive(const vespalib::string & index) const; }; -} // namespace index -} // namespace searchcorespi - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 32dc2531061..c75a74ff141 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -41,7 +41,6 @@ using vespalib::makeTask; using vespalib::string; using vespalib::Closure0; using vespalib::Executor; -using vespalib::LockGuard; using vespalib::Runnable; namespace searchcorespi::index { @@ -349,10 +348,12 @@ IndexMaintainer::loadDiskIndexes(const FusionSpec &spec, ISearchableIndexCollect namespace { + using LockGuard = std::lock_guard<std::mutex>; + ISearchableIndexCollection::SP getLeaf(const LockGuard &newSearchLock, const ISearchableIndexCollection::SP & is, bool warn=false) { - if (dynamic_cast<const WarmupIndexCollection *>(is.get()) != NULL) { + if (dynamic_cast<const WarmupIndexCollection *>(is.get()) != nullptr) { if (warn) { LOG(info, "Already warming up an index '%s'. Start using it immediately." " This is an indication that you have configured your warmup interval too long.", diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 671f87ff35b..5ff805b53f7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -18,9 +18,6 @@ #include <vespa/searchcorespi/flush/flushstats.h> #include <vespa/searchlib/attribute/fixedsourceselector.h> #include <vespa/searchlib/common/serialnum.h> -#include <vespa/vespalib/util/sync.h> -#include <memory> -#include <vector> namespace document { class Document; } @@ -74,6 +71,7 @@ class IndexMaintainer : public IIndexManager, using FlushIds = std::vector<uint32_t>; using FrozenMemoryIndexRefs = std::vector<FrozenMemoryIndexRef>; using ISourceSelector = search::queryeval::ISourceSelector; + using LockGuard = std::lock_guard<std::mutex>; const vespalib::string _base_dir; const WarmupConfig _warmupConfig; @@ -129,17 +127,17 @@ class IndexMaintainer : public IIndexManager, * and pruning of removed fields, since this will trigger more retries for * some of the operations. */ - vespalib::Lock _state_lock; // Outer lock (SL) - vespalib::Lock _index_update_lock; // Inner lock (IUL) - vespalib::Lock _new_search_lock; // Inner lock (NSL) - vespalib::Lock _remove_lock; // Lock for removing indexes. + std::mutex _state_lock; // Outer lock (SL) + mutable std::mutex _index_update_lock; // Inner lock (IUL) + mutable std::mutex _new_search_lock; // Inner lock (NSL) + std::mutex _remove_lock; // Lock for removing indexes. // Protected by SL + IUL - FusionSpec _fusion_spec; // Protected by FL - vespalib::Lock _fusion_lock; // Fusion spec lock (FL) + FusionSpec _fusion_spec; // Protected by FL + mutable std::mutex _fusion_lock; // Fusion spec lock (FL) uint32_t _maxFlushed; uint32_t _maxFrozen; ChangeGens _changeGens; // Protected by SL + IUL - vespalib::Lock _schemaUpdateLock; // Serialize rewrite of schema + std::mutex _schemaUpdateLock; // Serialize rewrite of schema const search::TuneFileAttributes _tuneFileAttributes; const IndexMaintainerContext _ctx; IIndexMaintainerOperations &_operations; @@ -179,8 +177,8 @@ class IndexMaintainer : public IIndexManager, ISearchableIndexCollection::UP loadDiskIndexes(const FusionSpec &spec, ISearchableIndexCollection::UP sourceList); void replaceSource(uint32_t sourceId, const IndexSearchable::SP &source); void appendSource(uint32_t sourceId, const IndexSearchable::SP &source); - void swapInNewIndex(vespalib::LockGuard & guard, ISearchableIndexCollection::SP indexes, IndexSearchable & source); - ISearchableIndexCollection::UP createNewSourceCollection(const vespalib::LockGuard &newSearchLock); + void swapInNewIndex(LockGuard & guard, ISearchableIndexCollection::SP indexes, IndexSearchable & source); + ISearchableIndexCollection::UP createNewSourceCollection(const LockGuard &newSearchLock); struct FlushArgs { IMemoryIndex::SP old_index; // Last memory index @@ -353,17 +351,17 @@ public: } IIndexCollection::SP getSourceCollection() const { - vespalib::LockGuard lock(_new_search_lock); + LockGuard lock(_new_search_lock); return _source_list; } searchcorespi::IndexSearchable::SP getSearchable() const override { - vespalib::LockGuard lock(_new_search_lock); + LockGuard lock(_new_search_lock); return _source_list; } search::SearchableStats getSearchableStats() const override { - vespalib::LockGuard lock(_new_search_lock); + LockGuard lock(_new_search_lock); return _source_list->getSearchableStats(); } diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/.gitignore b/searchcorespi/src/vespa/searchcorespi/plugin/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/plugin/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/CMakeLists.txt b/searchcorespi/src/vespa/searchcorespi/plugin/CMakeLists.txt deleted file mode 100644 index b564a593e4e..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/plugin/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcorespi_plugin OBJECT - SOURCES - factoryregistry.cpp - factoryloader.cpp - DEPENDS -) diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp b/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp deleted file mode 100644 index ba2b0b962f6..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "factoryloader.h" -#include <vespa/vespalib/util/exceptions.h> - -using vespalib::stringref; -using vespalib::make_string; -using vespalib::IllegalArgumentException; - -namespace searchcorespi { - -FactoryLoader::FactoryLoader() : - _libraries() -{ -} - -FactoryLoader::~FactoryLoader() = default; - -IIndexManagerFactory::UP -FactoryLoader::create(stringref factory) -{ - typedef IIndexManagerFactory* (*FuncT)(); - _libraries.loadLibrary(factory); - const FastOS_DynamicLibrary & lib = *_libraries.get(factory); - FuncT registrationMethod = reinterpret_cast<FuncT>(lib.GetSymbol("createIndexManagerFactory")); - if (registrationMethod == nullptr) { - throw IllegalArgumentException(make_string("Failed locating symbol 'createIndexManagerFactory' in library '%s' for factory '%s'.", - lib.GetLibName(), vespalib::string(factory).c_str())); - } - return IIndexManagerFactory::UP(registrationMethod()); -} - -} diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.h b/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.h deleted file mode 100644 index 3eda6557fa0..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> -#include <vespa/vespalib/util/librarypool.h> - -namespace searchcorespi { - -class FactoryLoader -{ -public: - FactoryLoader(); - ~FactoryLoader(); - /** - * Will load the library containing the factory. It will then locate the 'createIndexManagerFactory' - * symbol and run it to create the factory. - * @param the name of the library. Like 'vesparise'. - * @return the factory that is created. - */ - IIndexManagerFactory::UP create(vespalib::stringref factory); -private: - vespalib::LibraryPool _libraries; -}; - -} // namespace searchcorespi - diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/factoryregistry.cpp b/searchcorespi/src/vespa/searchcorespi/plugin/factoryregistry.cpp deleted file mode 100644 index bd1b914cf39..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/plugin/factoryregistry.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchcorespi/plugin/factoryregistry.h> -#include <vespa/vespalib/util/exceptions.h> - -using vespalib::LockGuard; -using vespalib::IllegalArgumentException; -using vespalib::stringref; -using vespalib::string; - -namespace searchcorespi { - -FactoryRegistry::FactoryRegistry() = default; - -FactoryRegistry::~FactoryRegistry() = default; - -void FactoryRegistry::add(stringref uniqueName, const IIndexManagerFactory::SP & factory) -{ - LockGuard guard(_lock); - if (_registry.find(uniqueName) == _registry.end()) { - _registry[uniqueName] = factory; - } else { - throw IllegalArgumentException("A factory is already registered with the same name as '" + uniqueName + "'.", VESPA_STRLOC); - } -} - -void FactoryRegistry::remove(stringref uniqueName) -{ - LockGuard guard(_lock); - if (_registry.find(uniqueName) == _registry.end()) { - throw IllegalArgumentException("No factory is registered with the name of '" + uniqueName + "'.", VESPA_STRLOC); - } - _registry.erase(uniqueName); -} - -const IIndexManagerFactory::SP & -FactoryRegistry::get(stringref uniqueName) const -{ - LockGuard guard(_lock); - Registry::const_iterator found = _registry.find(uniqueName); - if (found == _registry.end()) { - throw IllegalArgumentException("No factory is registered with the name of '" + uniqueName + "'.", VESPA_STRLOC); - } - return found->second; -} - -bool -FactoryRegistry::isRegistered(vespalib::stringref uniqueName) const -{ - LockGuard guard(_lock); - Registry::const_iterator found = _registry.find(uniqueName); - return found != _registry.end(); -} - -} diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/factoryregistry.h b/searchcorespi/src/vespa/searchcorespi/plugin/factoryregistry.h deleted file mode 100644 index feb27d62950..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/plugin/factoryregistry.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> - -namespace searchcorespi { - -/** - */ -class FactoryRegistry -{ -public: - FactoryRegistry(); - ~FactoryRegistry(); - /** - * This will register the plugged in factory under its official - * name. The plugin should call this method when it is loaded. E.g. - * by using either '__attribute__((constructor))' or using a - * global static object that will use its constructor to register - * the factory. - * @param uniqueName This is a name that is unique over all IndexManager factories. - * @param factory The factory instance for producing IndexManagers. - * @throws vespalib::IllegalArgument if factory is already registered. - */ - void add(vespalib::stringref uniqueName, const IIndexManagerFactory::SP & factory); - /** - * Will unregister a factory. Should be called when a sharedlibrary is being unloaded. - * @param uniqueName Unique name of factory to remove from registry. - * @throws vespalib::IllegalArgument if factory is already registered. - */ - void remove(vespalib::stringref uniqueName); - /** - * This method will fetch a factory given its unique name. - * @param name The name of the factory to return. - * @return The factory. - */ - const IIndexManagerFactory::SP & get(vespalib::stringref uniqueName) const; - /** - * Returns true if a factory with the given name has been registered. - */ - bool isRegistered(vespalib::stringref uniqueName) const; - -private: - typedef std::map<vespalib::string, IIndexManagerFactory::SP> Registry; - Registry _registry; - vespalib::Lock _lock; -}; - -} // namespace searchcorespi - diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/iindexmanagerfactory.h b/searchcorespi/src/vespa/searchcorespi/plugin/iindexmanagerfactory.h deleted file mode 100644 index c2eddc15cde..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/plugin/iindexmanagerfactory.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/searchcorespi/index/iindexmanager.h> -#include <vespa/searchcorespi/index/indexmaintainerconfig.h> -#include <vespa/searchcorespi/index/indexmaintainercontext.h> -#include <vespa/searchcorespi/index/indexmanagerconfig.h> -#include <vespa/config/configgen/configinstance.h> -#include <vespa/config/retriever/configsnapshot.h> -#include <vespa/config/retriever/configkeyset.h> - -namespace searchcorespi { - -/** - * Interface for an index manager factory. Every provider of an index manager is supposed to provide a - * factory for producing them. It is given the basedir, the schema and a collection of configs. - * The factory implementation must pick the config it needs and return an IIndexManager instance. - * The factory is registered by using the registerFactory() method. - */ -class IIndexManagerFactory -{ -public: - typedef std::shared_ptr<IIndexManagerFactory> SP; - typedef std::unique_ptr<IIndexManagerFactory> UP; - - virtual ~IIndexManagerFactory() {} - - /** - * This method will be called by a document db when it needs to create an index manager that - * uses an index maintainer (with source selector) in its implementation. - * The factory implementation must use RTTI to figure out what configs are what. - * It should receive all configs it needs, but wise to do sanity checking. - * - * @param managerConfig The config that will be used to construct an index manager. - * Note that if the factory used a different config id when populating the - * ConfigKeySet compared to the one in this config instance, it must - * also override the config id when fetching from the config snapshot. - * The root config received in the @ref getConfigKeys() call will also be - * part of the config snapshot in this config instance. - * @param maintainerConfig The config needed to construct an index maintainer. - * @param maintainerContext The context object used by an index maintainer during its lifetime. - * @return The index manager created or NULL if not, fx if configs are not as expected. - */ - virtual IIndexManager::UP createIndexManager(const IndexManagerConfig &managerConfig, - const index::IndexMaintainerConfig &maintainerConfig, - const index::IndexMaintainerContext &maintainerContext) = 0; - - /** - * The factory must return the set of config keys that it will require the config from. - * This will facilitate that the searchcore can fetch all configs needed in a pluggable way. - * - * @param configId The config id to use when generating the config keys. - * @param schema This is the initial index schema to be used. - * @param rootConfig This is an config instance that is the root config for the factory. - * Based on this config it must be able to tell if it needs any other config, - * and in that case provide the config keys. - * @return The set containing keys for all configs required. - */ - virtual config::ConfigKeySet getConfigKeys(const vespalib::string &configId, - const search::index::Schema &schema, - const config::ConfigInstance &rootConfig) = 0; -}; - -} // namespace searchcorespi - -extern "C" { -/** - * This is a method that each shared library must have in order provide a factory. - * This will be called by the one loading the library. - * @return The created factory that the caller will take ownership of. - */ -searchcorespi::IIndexManagerFactory * createIndexManagerFactory(); - -} - - diff --git a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp index 65b27dd411a..ae4b55948dc 100644 --- a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp +++ b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp @@ -62,8 +62,9 @@ struct InputInfo { std::vector<double> cmp_with; double usage_probability; double expected_usage; - InputInfo(vespalib::stringref name_in, double usage_probability_in, double expected_usage_in) - : name(name_in), cmp_with(), usage_probability(usage_probability_in), expected_usage(expected_usage_in) {} + InputInfo(vespalib::stringref name_in, double usage_probability_in, double expected_usage_in) noexcept + : name(name_in), cmp_with(), usage_probability(usage_probability_in), expected_usage(expected_usage_in) + {} double select_value() const { return cmp_with.empty() ? 0.5 : cmp_with[(cmp_with.size()-1)/2]; } diff --git a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp index 99b22e02e2e..5976b02c5cf 100644 --- a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp +++ b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp @@ -11,17 +11,17 @@ using search::attribute::PostingListMerger; struct Posting { uint32_t lid; int32_t weight; - Posting(uint32_t lid_, int32_t weight_) + Posting(uint32_t lid_, int32_t weight_) noexcept : lid(lid_), weight(weight_) { } - bool operator==(const Posting &rhs) const { + bool operator==(const Posting &rhs) const noexcept { return ((lid == rhs.lid) && (weight == rhs.weight)); } - bool operator<(const Posting &rhs) const { return lid < rhs.lid; } + bool operator<(const Posting &rhs) const noexcept { return lid < rhs.lid; } }; std::ostream &operator<<(std::ostream &os, const Posting &posting) @@ -35,11 +35,11 @@ class WeightedPostingList { std::vector<Posting> _entries; public: - WeightedPostingList(std::vector<Posting> entries) + WeightedPostingList(std::vector<Posting> entries) noexcept : _entries(std::move(entries)) { } - ~WeightedPostingList() { } + ~WeightedPostingList() = default; template <typename Func> void foreach(Func func) const { @@ -67,7 +67,7 @@ struct WeightedFixture { } - ~WeightedFixture() { } + ~WeightedFixture() = default; void reserveArray(uint32_t postingsCount, size_t postingsSize) { _merger.reserveArray(postingsCount, postingsSize); } diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 87aea2e3e8c..7328fe2c7ff 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -140,8 +140,9 @@ struct Result { uint32_t docid; double raw_score; int32_t match_weight; - Hit(uint32_t id, double raw, int32_t match_weight_in) - : docid(id), raw_score(raw), match_weight(match_weight_in) {} + Hit(uint32_t id, double raw, int32_t match_weight_in) noexcept + : docid(id), raw_score(raw), match_weight(match_weight_in) + {} }; size_t est_hits; bool est_empty; @@ -590,7 +591,7 @@ TEST("require that attribute weighted set term works") { TEST("require that predicate query in non-predicate field yields empty.") { MyAttributeManager attribute_manager = makeAttributeManager("foo"); - PredicateQueryTerm::UP term(new PredicateQueryTerm); + auto term = std::make_unique<PredicateQueryTerm>(); SimplePredicateQuery node(std::move(term), field, 0, Weight(1)); Result result = do_search(attribute_manager, node, true); EXPECT_TRUE(result.est_empty); @@ -605,7 +606,7 @@ TEST("require that predicate query in predicate field yields results.") { const_cast<PredicateAttribute::IntervalRange *>(attr->getIntervalRangeVector())[2] = 1u; MyAttributeManager attribute_manager(attr); - PredicateQueryTerm::UP term(new PredicateQueryTerm); + auto term = std::make_unique<PredicateQueryTerm>(); SimplePredicateQuery node(std::move(term), field, 0, Weight(1)); Result result = do_search(attribute_manager, node, true); EXPECT_FALSE(result.est_empty); diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 70c23dc4191..1e99221551f 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -21,7 +21,7 @@ using common::FileHeaderContext; using vespalib::ThreadStackExecutor; struct MyFileHeaderContext : public FileHeaderContext { - virtual void addTags(vespalib::GenericHeader &header, const vespalib::string &name) const override { + void addTags(vespalib::GenericHeader &header, const vespalib::string &name) const override { (void) header; (void) name; } @@ -29,7 +29,7 @@ struct MyFileHeaderContext : public FileHeaderContext { struct SetLidObserver : public ISetLid { std::vector<uint32_t> lids; - virtual void setLid(const vespalib::LockGuard &guard, uint32_t lid, const LidInfo &lidInfo) override { + void setLid(const LockGuard &guard, uint32_t lid, const LidInfo &lidInfo) override { (void) guard; (void) lidInfo; lids.push_back(lid); @@ -38,12 +38,12 @@ struct SetLidObserver : public ISetLid { struct BucketizerObserver : public IBucketizer { mutable std::vector<uint32_t> lids; - virtual document::BucketId getBucketOf(const vespalib::GenerationHandler::Guard &guard, uint32_t lid) const override { + document::BucketId getBucketOf(const vespalib::GenerationHandler::Guard &guard, uint32_t lid) const override { (void) guard; lids.push_back(lid); return document::BucketId(); } - virtual vespalib::GenerationHandler::Guard getGuard() const override { + vespalib::GenerationHandler::Guard getGuard() const override { return vespalib::GenerationHandler::Guard(); } }; @@ -62,7 +62,7 @@ struct FixtureBase { uint64_t serialNum; TuneFileSummary tuneFile; MyFileHeaderContext fileHeaderCtx; - vespalib::Lock updateLock; + std::mutex updateLock; SetLidObserver lidObserver; BucketizerObserver bucketizer; @@ -70,8 +70,7 @@ struct FixtureBase { return serialNum++; }; - FixtureBase(const vespalib::string &baseName, - bool dirCleanup = true) + explicit FixtureBase(const vespalib::string &baseName, bool dirCleanup = true) : dir(baseName), executor(1, 0x10000), serialNum(1), @@ -83,20 +82,21 @@ struct FixtureBase { { dir.cleanup(dirCleanup); } - ~FixtureBase() {} - void assertLidMap(const std::vector<uint32_t> &expLids) { + ~FixtureBase(); + void assertLidMap(const std::vector<uint32_t> &expLids) const { EXPECT_EQUAL(expLids, lidObserver.lids); } - void assertBucketizer(const std::vector<uint32_t> &expLids) { + void assertBucketizer(const std::vector<uint32_t> &expLids) const { EXPECT_EQUAL(expLids, bucketizer.lids); } }; +FixtureBase::~FixtureBase() = default; + struct ReadFixture : public FixtureBase { FileChunk chunk; - ReadFixture(const vespalib::string &baseName, - bool dirCleanup = true) + explicit ReadFixture(const vespalib::string &baseName, bool dirCleanup = true) : FixtureBase(baseName, dirCleanup), chunk(FileChunk::FileId(0), FileChunk::NameId(1234), @@ -108,7 +108,7 @@ struct ReadFixture : public FixtureBase { dir.cleanup(dirCleanup); } void updateLidMap(uint32_t docIdLimit) { - vespalib::LockGuard guard(updateLock); + std::unique_lock guard(updateLock); chunk.updateLidMap(guard, lidObserver, serialNum, docIdLimit); } }; @@ -145,7 +145,7 @@ struct WriteFixture : public FixtureBase { return *this; } void updateLidMap(uint32_t docIdLimit) { - vespalib::LockGuard guard(updateLock); + std::unique_lock guard(updateLock); chunk.updateLidMap(guard, lidObserver, serialNum, docIdLimit); serialNum = chunk.getSerialNum(); } diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index 5592edcc514..ab45ca0bdbe 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -114,7 +114,7 @@ AttributeManagerFixture::buildAttribute(const vespalib::string &name, BasicType auto attr = std::dynamic_pointer_cast<AttributeType>(attrBase); EXPECT_TRUE(attr); attr->addReservedDoc(); - for (const auto &value : values) { + for (const std::conditional_t<std::is_same_v<bool, ValueType>, bool, ValueType&> value : values) { uint32_t docId = 0; EXPECT_TRUE(attr->addDoc(docId)); EXPECT_NOT_EQUAL(0u, docId); diff --git a/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp index af7ea9be481..584a8121b16 100644 --- a/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp @@ -16,10 +16,10 @@ using namespace search::memoryindex; struct WordFieldPair { vespalib::string _word; uint32_t _fieldId; - WordFieldPair(vespalib::stringref word, uint32_t fieldId) + WordFieldPair(vespalib::stringref word, uint32_t fieldId) noexcept : _word(word), _fieldId(fieldId) {} - bool operator<(const WordFieldPair &rhs) const { + bool operator<(const WordFieldPair &rhs) const noexcept { if (_word != rhs._word) { return _word < rhs._word; } diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index 269600d26d4..5a5a5eafb2c 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -38,7 +38,7 @@ const uint32_t x_aspect = 0; const Location location(position, max_distance, x_aspect); PredicateQueryTerm::UP getPredicateQueryTerm() { - PredicateQueryTerm::UP pqt(new PredicateQueryTerm); + auto pqt = std::make_unique<PredicateQueryTerm>(); pqt->addFeature("key", "value"); pqt->addRangeFeature("key2", 42, 0xfff); return pqt; @@ -242,7 +242,7 @@ void checkQueryTreeTypes(Node *node) { EXPECT_TRUE(checkTerm(string_term, str[5], view[5], id[5], weight[5])); auto* predicateQuery = as_node<PredicateQuery>(and_node->getChildren()[5]); - PredicateQueryTerm::UP pqt(new PredicateQueryTerm); + auto pqt = std::make_unique<PredicateQueryTerm>(); EXPECT_TRUE(checkTerm(predicateQuery, getPredicateQueryTerm(), view[3], id[3], weight[3])); auto* dotProduct = as_node<DotProduct>(and_node->getChildren()[6]); diff --git a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp index 28b0d103040..3dd2ec26dea 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp @@ -40,9 +40,9 @@ struct Fixture { Fixture() : field(42, 0), - attribute(new PredicateAttribute("f", attribute::Config(attribute::BasicType::PREDICATE))), - query(PredicateQueryTerm::UP(new PredicateQueryTerm), - "view", 0, Weight(1)) { + attribute(std::make_shared<PredicateAttribute>("f", attribute::Config(attribute::BasicType::PREDICATE))), + query(std::make_unique<PredicateQueryTerm>(),"view", 0, Weight(1)) + { query.getTerm()->addFeature("key", "value"); query.getTerm()->addRangeFeature("range_key", 42); } @@ -219,7 +219,7 @@ TEST_F("require that blueprint can set up search with subqueries", Fixture) { std::vector<Interval>{{0x0002ffff}}; f.indexDocument(doc_id, annotations); - SimplePredicateQuery query(PredicateQueryTerm::UP(new PredicateQueryTerm), + SimplePredicateQuery query(std::make_unique<PredicateQueryTerm>(), "view", 0, Weight(1)); query.getTerm()->addFeature("key", "value", 1); query.getTerm()->addFeature("key2", "value", 2); diff --git a/searchlib/src/tests/transactionlog/chunks_test.cpp b/searchlib/src/tests/transactionlog/chunks_test.cpp index 4a74dd7f5bc..a3cf9f8bd92 100644 --- a/searchlib/src/tests/transactionlog/chunks_test.cpp +++ b/searchlib/src/tests/transactionlog/chunks_test.cpp @@ -79,7 +79,7 @@ TEST("test empty commitchunk") { struct Counter : public search::IDestructorCallback { std::atomic<uint32_t> & _counter; - Counter(std::atomic<uint32_t> & counter) : _counter(counter) { _counter++; } + Counter(std::atomic<uint32_t> & counter) noexcept : _counter(counter) { _counter++; } ~Counter() override { _counter--; } }; diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index fffb70467a3..e097eebd42c 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -311,7 +311,7 @@ using Counter = std::atomic<size_t>; class CountDone : public IDestructorCallback { public: - explicit CountDone(Counter & inFlight) : _inFlight(inFlight) { ++_inFlight; } + explicit CountDone(Counter & inFlight) noexcept : _inFlight(inFlight) { ++_inFlight; } ~CountDone() override { --_inFlight; } private: Counter & _inFlight; diff --git a/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h index a889120f8df..1cd07cf4e23 100644 --- a/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h +++ b/searchlib/src/vespa/searchlib/attribute/bitvector_search_cache.h @@ -8,11 +8,8 @@ #include <memory> #include <mutex> -namespace search { - -class BitVector; - -namespace attribute { +namespace search { class BitVector; } +namespace search::attribute { /** * Class that caches posting lists (as bit vectors) for a set of search terms. @@ -31,7 +28,7 @@ public: ReadGuardUP dmsReadGuard; BitVectorSP bitVector; uint32_t docIdLimit; - Entry(ReadGuardUP dmsReadGuard_, BitVectorSP bitVector_, uint32_t docIdLimit_) + Entry(ReadGuardUP dmsReadGuard_, BitVectorSP bitVector_, uint32_t docIdLimit_) noexcept : dmsReadGuard(std::move(dmsReadGuard_)), bitVector(std::move(bitVector_)), docIdLimit(docIdLimit_) {} }; @@ -52,4 +49,3 @@ public: }; } -} diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index bc0d965bcc1..9c40de26db6 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -93,7 +93,7 @@ struct WeightedRef { EntryRef revMapIdx; int32_t weight; - WeightedRef(EntryRef revMapIdx_, int32_t weight_) + WeightedRef(EntryRef revMapIdx_, int32_t weight_) noexcept : revMapIdx(revMapIdx_), weight(weight_) { @@ -118,11 +118,11 @@ class ReverseMappingBitVector const ReverseMapping &_reverseMapping; EntryRef _revMapIdx; public: - ReverseMappingBitVector(const ReverseMapping &reverseMapping, EntryRef revMapIdx) + ReverseMappingBitVector(const ReverseMapping &reverseMapping, EntryRef revMapIdx) noexcept : _reverseMapping(reverseMapping), _revMapIdx(revMapIdx) {} - ~ReverseMappingBitVector() { } + ~ReverseMappingBitVector() = default; template <typename Func> void foreach_key(Func func) const { diff --git a/searchlib/src/vespa/searchlib/attribute/interlock.h b/searchlib/src/vespa/searchlib/attribute/interlock.h index cf9298686c3..30efeaf32fd 100644 --- a/searchlib/src/vespa/searchlib/attribute/interlock.h +++ b/searchlib/src/vespa/searchlib/attribute/interlock.h @@ -4,11 +4,7 @@ #include <mutex> -namespace search -{ - -namespace attribute -{ +namespace search::attribute { class InterlockGuard; @@ -35,7 +31,7 @@ class Interlock { std::mutex _mutex; friend class InterlockGuard; public: - Interlock() + Interlock() noexcept : _mutex() { } @@ -60,6 +56,4 @@ public: }; -} - -} +}
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/attribute/multivalue.h b/searchlib/src/vespa/searchlib/attribute/multivalue.h index c59f975e00a..54c10bdc7cf 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalue.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalue.h @@ -10,12 +10,9 @@ template <typename T> class Value { public: typedef T ValueType; - Value() - : _v() - { - } - Value(T v) : _v(v) { } - Value(T v, int32_t w) : _v(v) { (void) w; } + Value() noexcept : _v() {} + Value(T v) noexcept : _v(v) { } + Value(T v, int32_t w) noexcept : _v(v) { (void) w; } T value() const { return _v; } operator T () const { return _v; } operator T & () { return _v; } @@ -36,8 +33,8 @@ template <typename T> class WeightedValue { public: typedef T ValueType; - WeightedValue() : _v(), _w(1) { } - WeightedValue(T v, int32_t w) : _v(v), _w(w) { } + WeightedValue() noexcept : _v(), _w(1) { } + WeightedValue(T v, int32_t w) noexcept : _v(v), _w(w) { } T value() const { return _v; } operator T () const { return _v; } operator T & () { return _v; } diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocc_field_params.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocc_field_params.cpp index b789bf16947..751c1db50b2 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocc_field_params.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocc_field_params.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/index/postinglistparams.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".posocc_field_params"); diff --git a/searchlib/src/vespa/searchlib/common/gatecallback.h b/searchlib/src/vespa/searchlib/common/gatecallback.h index 1e85d796089..b6e7690f820 100644 --- a/searchlib/src/vespa/searchlib/common/gatecallback.h +++ b/searchlib/src/vespa/searchlib/common/gatecallback.h @@ -9,7 +9,7 @@ namespace search { class GateCallback : public IDestructorCallback { public: - GateCallback(vespalib::Gate & gate) : _gate(gate) {} + GateCallback(vespalib::Gate & gate) noexcept : _gate(gate) {} ~GateCallback() override; private: vespalib::Gate & _gate; @@ -17,7 +17,7 @@ private: class IgnoreCallback : public IDestructorCallback { public: - IgnoreCallback() { } + IgnoreCallback() noexcept { } ~IgnoreCallback() override = default; }; diff --git a/searchlib/src/vespa/searchlib/common/lid_usage_stats.h b/searchlib/src/vespa/searchlib/common/lid_usage_stats.h index 1dd3881892f..97e16ffe768 100644 --- a/searchlib/src/vespa/searchlib/common/lid_usage_stats.h +++ b/searchlib/src/vespa/searchlib/common/lid_usage_stats.h @@ -20,7 +20,7 @@ private: uint32_t _highestUsedLid; public: - LidUsageStats() + LidUsageStats() noexcept : _lidLimit(0), _usedLids(0), _lowestFreeLid(0), @@ -30,7 +30,7 @@ public: LidUsageStats(uint32_t lidLimit, uint32_t usedLids, uint32_t lowestFreeLid, - uint32_t highestUsedLid) + uint32_t highestUsedLid) noexcept : _lidLimit(lidLimit), _usedLids(usedLids), _lowestFreeLid(lowestFreeLid), diff --git a/searchlib/src/vespa/searchlib/common/rankedhit.h b/searchlib/src/vespa/searchlib/common/rankedhit.h index 635f6e350a5..8a0efb4ce3c 100644 --- a/searchlib/src/vespa/searchlib/common/rankedhit.h +++ b/searchlib/src/vespa/searchlib/common/rankedhit.h @@ -9,8 +9,8 @@ namespace search { struct RankedHit { - RankedHit() : _docId(0), _rankValue(zero_rank_value) { } - RankedHit(unsigned int docId, HitRank rank = zero_rank_value) : _docId(docId), _rankValue(rank) { } + RankedHit() noexcept : _docId(0), _rankValue(zero_rank_value) { } + RankedHit(unsigned int docId, HitRank rank = zero_rank_value) noexcept : _docId(docId), _rankValue(rank) { } unsigned int getDocId() const { return _docId & 0x7fffffff; } bool hasMore() const { return _docId & 0x80000000; } HitRank getRank() const { return _rankValue; } @@ -21,13 +21,13 @@ struct RankedHit { class RankedHitIterator { public: - RankedHitIterator(const RankedHit * h, size_t sz) : _h(h), _sz(sz), _pos(0) { } - bool hasNext() const { return _pos < _sz; } - uint32_t next() { return _h[_pos++].getDocId(); } + RankedHitIterator(const RankedHit * h, size_t sz) noexcept : _h(h), _sz(sz), _pos(0) { } + bool hasNext() const noexcept { return _pos < _sz; } + uint32_t next() noexcept { return _h[_pos++].getDocId(); } private: const RankedHit *_h; - const size_t _sz; - size_t _pos; + const size_t _sz; + size_t _pos; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/scheduletaskcallback.h b/searchlib/src/vespa/searchlib/common/scheduletaskcallback.h index 27bbe751532..00f0d4b29f5 100644 --- a/searchlib/src/vespa/searchlib/common/scheduletaskcallback.h +++ b/searchlib/src/vespa/searchlib/common/scheduletaskcallback.h @@ -4,8 +4,7 @@ #include "idestructorcallback.h" #include <vespa/vespalib/util/executor.h> -namespace search -{ +namespace search { /** * Class that schedules a task when instance is destroyed. Typically a @@ -20,12 +19,11 @@ class ScheduleTaskCallback : public IDestructorCallback vespalib::Executor::Task::UP _task; public: ScheduleTaskCallback(vespalib::Executor &executor, - vespalib::Executor::Task::UP task) + vespalib::Executor::Task::UP task) noexcept : _executor(executor), _task(std::move(task)) - { - } - virtual ~ScheduleTaskCallback() { + {} + ~ScheduleTaskCallback() override { _executor.execute(std::move(_task)); } }; diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.h b/searchlib/src/vespa/searchlib/common/tunefileinfo.h index bcd6765845b..e27290d35de 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.h +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.h @@ -19,7 +19,7 @@ private: TuneControl _tuneControl; public: - TuneFileSeqRead() : _tuneControl(NORMAL) { } + TuneFileSeqRead() noexcept : _tuneControl(NORMAL) { } void setWantDirectIO() { _tuneControl = DIRECTIO; } bool getWantDirectIO() const { return _tuneControl == DIRECTIO; } @@ -62,7 +62,7 @@ private: TuneControl _tuneControl; public: - TuneFileSeqWrite() : _tuneControl(NORMAL) { } + TuneFileSeqWrite() noexcept : _tuneControl(NORMAL) { } void setWantDirectIO() { _tuneControl = DIRECTIO; } bool getWantDirectIO() const { return _tuneControl == DIRECTIO; } bool getWantSyncWrites() const { return _tuneControl == OSYNC; } @@ -99,7 +99,7 @@ private: int _mmapFlags; int _advise; public: - TuneFileRandRead() + TuneFileRandRead() noexcept : _tuneControl(NORMAL), _mmapFlags(0), _advise(0) @@ -139,9 +139,9 @@ public: TuneFileSeqRead _read; TuneFileSeqWrite _write; - TuneFileIndexing() : _read(), _write() {} + TuneFileIndexing() noexcept : _read(), _write() {} - TuneFileIndexing(const TuneFileSeqRead &r, const TuneFileSeqWrite &w) : _read(r), _write(w) { } + TuneFileIndexing(const TuneFileSeqRead &r, const TuneFileSeqWrite &w) noexcept : _read(r), _write(w) { } bool operator==(const TuneFileIndexing &rhs) const { return _read == rhs._read && _write == rhs._write; @@ -161,8 +161,8 @@ class TuneFileSearch public: TuneFileRandRead _read; - TuneFileSearch() : _read() { } - TuneFileSearch(const TuneFileRandRead &r) : _read(r) { } + TuneFileSearch() noexcept : _read() { } + TuneFileSearch(const TuneFileRandRead &r) noexcept : _read(r) { } bool operator==(const TuneFileSearch &rhs) const { return _read == rhs._read; } bool operator!=(const TuneFileSearch &rhs) const { return _read != rhs._read; } }; @@ -178,7 +178,7 @@ public: TuneFileIndexing _indexing; TuneFileSearch _search; - TuneFileIndexManager() : _indexing(), _search() { } + TuneFileIndexManager() noexcept : _indexing(), _search() { } bool operator==(const TuneFileIndexManager &rhs) const { return _indexing == rhs._indexing && _search == rhs._search; @@ -198,7 +198,7 @@ class TuneFileAttributes public: TuneFileSeqWrite _write; - TuneFileAttributes() : _write() { } + TuneFileAttributes() noexcept : _write() { } bool operator==(const TuneFileAttributes &rhs) const { return _write == rhs._write; @@ -220,7 +220,7 @@ public: TuneFileSeqWrite _write; TuneFileRandRead _randRead; - TuneFileSummary() : _seqRead(), _write(), _randRead() { } + TuneFileSummary() noexcept : _seqRead(), _write(), _randRead() { } bool operator==(const TuneFileSummary &rhs) const { return _seqRead == rhs._seqRead && @@ -248,7 +248,7 @@ public: TuneFileAttributes _attr; TuneFileSummary _summary; - TuneFileDocumentDB() : _index(), _attr(), _summary() { } + TuneFileDocumentDB() noexcept : _index(), _attr(), _summary() { } bool operator==(const TuneFileDocumentDB &rhs) const { return _index == rhs._index && diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h index e9b1efa5c7d..4e71a8356c0 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h @@ -24,7 +24,7 @@ public: uint32_t _field_length; uint32_t _num_occs; uint32_t _features_size; - DocIdAndFeatureSize(uint32_t doc_id, uint32_t field_length, uint32_t num_occs, uint32_t features_size) + DocIdAndFeatureSize(uint32_t doc_id, uint32_t field_length, uint32_t num_occs, uint32_t features_size) noexcept : _doc_id(doc_id), _field_length(field_length), _num_occs(num_occs), diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h index cf059b6cb04..52691d2c415 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.h +++ b/searchlib/src/vespa/searchlib/docstore/compacter.h @@ -51,7 +51,7 @@ private: uint64_t _writeCount; vespalib::duration _maxBucketGuardDuration; vespalib::steady_time _lastSample; - vespalib::Lock _lock; + std::mutex _lock; vespalib::MemoryDataStore _backingMemory; std::vector<StoreByBucket> _tmpStore; GenerationHandler::Guard _lidGuard; diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index b4ef45187ee..d66e178717c 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -320,7 +320,7 @@ appendChunks(FixedParams * args, Chunk::UP chunk) for (const Chunk::Entry & e : ll) { LidInfo lidInfo(args->fileId, chunk->getId(), e.netSize()); if (args->db.getLid(args->lidReadGuard, e.getLid()) == lidInfo) { - vespalib::LockGuard guard(args->db.getLidGuard(e.getLid())); + auto guard(args->db.getLidGuard(e.getLid())); if (args->db.getLid(args->lidReadGuard, e.getLid()) == lidInfo) { // I am still in use so I need to taken care of. vespalib::ConstBufferRef data(chunk->getLid(e.getLid())); diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h index b68db801d60..3febb51ca69 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h @@ -9,7 +9,6 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/util/memoryusage.h> #include <vespa/vespalib/util/ptrholder.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/time.h> @@ -30,9 +29,9 @@ class IWriteData { public: typedef std::unique_ptr<IWriteData> UP; - using LockGuard = vespalib::LockGuard; + using LockGuard = std::unique_lock<std::mutex>; - virtual ~IWriteData() { } + virtual ~IWriteData() = default; virtual void write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) = 0; virtual void close() = 0; @@ -41,7 +40,7 @@ public: class IFileChunkVisitorProgress { public: - virtual ~IFileChunkVisitorProgress() { } + virtual ~IFileChunkVisitorProgress() = default; virtual void updateProgress() = 0; }; @@ -73,10 +72,10 @@ private: class FileChunk { public: - using LockGuard = vespalib::LockGuard; + using LockGuard = std::unique_lock<std::mutex>; class NameId { public: - explicit NameId(size_t id) : _id(id) { } + explicit NameId(size_t id) noexcept : _id(id) { } uint64_t getId() const { return _id; } vespalib::string createName(const vespalib::string &baseName) const; bool operator == (const NameId & rhs) const { return _id == rhs._id; } @@ -90,7 +89,7 @@ public: }; class FileId { public: - explicit FileId(uint32_t id) : _id(id) { } + explicit FileId(uint32_t id) noexcept : _id(id) { } uint32_t getId() const { return _id; } bool operator != (const FileId & rhs) const { return _id != rhs._id; } bool operator == (const FileId & rhs) const { return _id == rhs._id; } diff --git a/searchlib/src/vespa/searchlib/docstore/lid_info.h b/searchlib/src/vespa/searchlib/docstore/lid_info.h index 10ddd868c41..152246631a7 100644 --- a/searchlib/src/vespa/searchlib/docstore/lid_info.h +++ b/searchlib/src/vespa/searchlib/docstore/lid_info.h @@ -3,16 +3,16 @@ #pragma once #include <vespa/vespalib/util/generationhandler.h> -#include <vespa/vespalib/util/sync.h> #include <limits> #include <vector> +#include <mutex> namespace search { class LidInfo { public: - LidInfo() : _value() { } - LidInfo(uint64_t rep) { _value.r = rep; } + LidInfo() noexcept : _value() { } + LidInfo(uint64_t rep) noexcept { _value.r = rep; } LidInfo(uint32_t fileId, uint32_t chunkId, uint32_t size); uint32_t getFileId() const { return _value.v.fileId; } uint32_t getChunkId() const { return _value.v.chunkId; } @@ -57,7 +57,7 @@ private: class LidInfoWithLid : public LidInfo { public: - LidInfoWithLid(LidInfo lidInfo, uint32_t lid) : LidInfo(lidInfo), _lid(lid) { } + LidInfoWithLid(LidInfo lidInfo, uint32_t lid) noexcept : LidInfo(lidInfo), _lid(lid) { } uint32_t getLid() const { return _lid; } private: uint32_t _lid; @@ -68,8 +68,8 @@ typedef std::vector<LidInfoWithLid> LidInfoWithLidV; class ISetLid { public: - using LockGuard = vespalib::LockGuard; - virtual ~ISetLid() { } + using LockGuard = std::unique_lock<std::mutex>; + virtual ~ISetLid() = default; virtual void setLid(const LockGuard & guard, uint32_t lid, const LidInfo & lm) = 0; }; @@ -77,10 +77,11 @@ class IGetLid { public: using Guard = vespalib::GenerationHandler::Guard; - virtual ~IGetLid() { } + using LockGuard = std::unique_lock<std::mutex>; + virtual ~IGetLid() = default; virtual LidInfo getLid(const Guard & guard, uint32_t lid) const = 0; - virtual vespalib::LockGuard getLidGuard(uint32_t lid) const = 0; + virtual LockGuard getLidGuard(uint32_t lid) const = 0; virtual Guard getLidReadGuard() const = 0; }; diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 4adb3507eeb..37510afc572 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -464,8 +464,7 @@ bool LogDataStore::shouldCompactToActiveFile(size_t compactedSize) const { void LogDataStore::setNewFileChunk(const LockGuard & guard, FileChunk::UP file) { - (void) guard; - assert(guard.locks(_updateLock)); + assert(hasUpdateLock(guard)); size_t fileId = file->getFileId().getId(); assert( ! _fileChunks[fileId]); _fileChunks[fileId] = std::move(file); @@ -854,7 +853,7 @@ LogDataStore::findIncompleteCompactedFiles(const NameIdSet & partList) { LogDataStore::NameIdSet LogDataStore::getAllActiveFiles() const { NameIdSet files; - vespalib::LockGuard guard(_updateLock); + LockGuard guard(_updateLock); for (const auto & fc : _fileChunks) { if (fc) { files.insert(fc->getNameId()); @@ -1219,7 +1218,7 @@ LogDataStore::canShrinkLidSpace() const } bool -LogDataStore::canShrinkLidSpace(const vespalib::LockGuard &) const +LogDataStore::canShrinkLidSpace(const LockGuard &) const { return getDocIdLimit() < _lidInfo.size() && _compactLidSpaceGeneration < _genHandler.getFirstUsedGeneration(); diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index c709c607f37..12073c49cde 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -33,7 +33,7 @@ private: using FileId = FileChunk::FileId; public: using NameIdSet = std::set<NameId>; - using LockGuard = vespalib::LockGuard; + using LockGuard = std::unique_lock<std::mutex>; using CompressionConfig = vespalib::compression::CompressionConfig; class Config { public: @@ -146,9 +146,9 @@ public: } // Implements IGetLid API - LockGuard getLidGuard(uint32_t lid) const override { + IGetLid::LockGuard getLidGuard(uint32_t lid) const override { (void) lid; - return LockGuard(_updateLock); + return IGetLid::LockGuard(_updateLock); } // Implements IGetLid API @@ -160,11 +160,14 @@ public: return LidInfo(); } } - FileId getActiveFileId(const vespalib::LockGuard & guard) const { - assert(guard.locks(_updateLock)); + FileId getActiveFileId(const LockGuard & guard) const { + assert(hasUpdateLock(guard)); (void) guard; return _active; } + bool hasUpdateLock(const LockGuard & guard) const { + return (guard.mutex() == &_updateLock) && guard.owns_lock(); + } DataStoreStorageStats getStorageStats() const override; vespalib::MemoryUsage getMemoryUsage() const override; @@ -185,7 +188,7 @@ private: class FileChunkHolder; // Implements ISetLid API - void setLid(const LockGuard & guard, uint32_t lid, const LidInfo & lm) override; + void setLid(const ISetLid::LockGuard & guard, uint32_t lid, const LidInfo & lm) override; void compactWorst(double bloatLimit, double spreadLimit, bool prioritizeDiskBloat); void compactFile(FileId chunkId); @@ -211,25 +214,21 @@ private: vespalib::string ls(const NameIdSet & partList); WriteableFileChunk & getActive(const LockGuard & guard) { - assert(guard.locks(_updateLock)); - (void) guard; + assert(hasUpdateLock(guard)); return static_cast<WriteableFileChunk &>(*_fileChunks[_active.getId()]); } const WriteableFileChunk & getActive(const LockGuard & guard) const { - assert(guard.locks(_updateLock)); - (void) guard; + assert(hasUpdateLock(guard)); return static_cast<const WriteableFileChunk &>(*_fileChunks[_active.getId()]); } const FileChunk * getPrevActive(const LockGuard & guard) const { - assert(guard.locks(_updateLock)); - (void) guard; + assert(hasUpdateLock(guard)); return ( !_prevActive.isActive() ) ? _fileChunks[_prevActive.getId()].get() : nullptr; } void setActive(const LockGuard & guard, FileId fileId) { - assert(guard.locks(_updateLock)); - (void) guard; + assert(hasUpdateLock(guard)); _prevActive = _active; _active = fileId; } @@ -270,7 +269,7 @@ private: bool shouldCompactToActiveFile(size_t compactedSize) const; std::pair<bool, FileId> findNextToCompact(double bloatLimit, double spreadLimit, bool prioritizeDiskBloat); void incGeneration(); - bool canShrinkLidSpace(const vespalib::LockGuard &guard) const; + bool canShrinkLidSpace(const LockGuard &guard) const; typedef std::vector<FileId> FileIdxVector; Config _config; @@ -282,7 +281,7 @@ private: std::vector<uint32_t> _holdFileChunks; FileId _active; FileId _prevActive; - vespalib::Lock _updateLock; + mutable std::mutex _updateLock; bool _readOnly; vespalib::ThreadExecutor &_executor; SerialNum _initFlushSyncToken; diff --git a/searchlib/src/vespa/searchlib/docstore/randreaders.cpp b/searchlib/src/vespa/searchlib/docstore/randreaders.cpp index 2eb419cf8eb..cb670f513f7 100644 --- a/searchlib/src/vespa/searchlib/docstore/randreaders.cpp +++ b/searchlib/src/vespa/searchlib/docstore/randreaders.cpp @@ -107,7 +107,7 @@ MMapRandReadDynamic::MMapRandReadDynamic(const vespalib::string &fileName, int m void MMapRandReadDynamic::remap(size_t sz) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if ((sz > 0) && _holder.hasValue() && contains(*_holder.get(), sz)) { return; } diff --git a/searchlib/src/vespa/searchlib/docstore/randreaders.h b/searchlib/src/vespa/searchlib/docstore/randreaders.h index 4503c3600d4..00dc371bddf 100644 --- a/searchlib/src/vespa/searchlib/docstore/randreaders.h +++ b/searchlib/src/vespa/searchlib/docstore/randreaders.h @@ -47,7 +47,7 @@ private: vespalib::PtrHolder<FastOS_FileInterface> _holder; int _mmapFlags; int _fadviseOptions; - vespalib::Lock _lock; + std::mutex _lock; }; class NormalRandRead : public FileRandRead diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp index 5e595d0bb14..d7711b61d78 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp @@ -12,7 +12,7 @@ using document::BucketId; using vespalib::makeTask; using vespalib::makeClosure; -StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression) +StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression) noexcept : _chunkSerial(0), _current(), _where(), diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h index 8be0610b588..1365dcb4416 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h @@ -24,9 +24,12 @@ class StoreByBucket using ConstBufferRef = vespalib::ConstBufferRef; using CompressionConfig = vespalib::compression::CompressionConfig; public: - StoreByBucket(vespalib::MemoryDataStore & backingMemory, const CompressionConfig & compression); - StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression); - StoreByBucket(StoreByBucket &&) = default; + StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression) noexcept; + //TODO Putting the below move constructor into cpp file fails for some unknown reason. Needs to be resolved. + StoreByBucket(StoreByBucket &&) noexcept = default; + StoreByBucket(const StoreByBucket &) = delete; + StoreByBucket & operator=(StoreByBucket &&) noexcept = delete; + StoreByBucket & operator = (const StoreByBucket &) = delete; ~StoreByBucket(); class IWrite { public: @@ -52,10 +55,10 @@ private: void closeChunk(Chunk::UP chunk); struct Index { using BucketId=document::BucketId; - Index(BucketId bucketId, uint32_t id, uint32_t chunkId, uint32_t entry) : + Index(BucketId bucketId, uint32_t id, uint32_t chunkId, uint32_t entry) noexcept : _bucketId(bucketId), _id(id), _chunkId(chunkId), _lid(entry) { } - bool operator < (const Index & b) const { + bool operator < (const Index & b) const noexcept { return BucketId::bucketIdToKey(_bucketId.getRawId()) < BucketId::bucketIdToKey(b._bucketId.getRawId()); } BucketId _bucketId; diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 6990a0a3ed7..1671b6cae69 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -11,7 +11,6 @@ namespace search::docstore { using vespalib::ConstBufferRef; -using vespalib::LockGuard; using vespalib::DataBuffer; using vespalib::alloc::Alloc; using vespalib::alloc::MemoryAllocator; @@ -167,7 +166,7 @@ VisitCache::reconfigure(size_t cacheSize, const CompressionConfig &compression) VisitCache::Cache::IdSet -VisitCache::Cache::findSetsContaining(const LockGuard &, const KeySet & keys) const { +VisitCache::Cache::findSetsContaining(const UniqueLock &, const KeySet & keys) const { IdSet found; for (uint32_t subKey : keys.getKeys()) { const auto foundLid = _lid2Id.find(subKey); @@ -194,7 +193,7 @@ VisitCache::Cache::readSet(const KeySet & key) } void -VisitCache::Cache::locateAndInvalidateOtherSubsets(const LockGuard & cacheGuard, const KeySet & keys) +VisitCache::Cache::locateAndInvalidateOtherSubsets(const UniqueLock & cacheGuard, const KeySet & keys) { // Due to the implementation of insert where the global lock is released and the fact // that 2 overlapping keysets kan have different keys and use different ValueLock diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index 8a06794ee35..430481acd4c 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -23,7 +23,7 @@ public: KeySet() : _keys() { } KeySet(uint32_t key); explicit KeySet(const IDocumentStore::LidVector &keys); - uint32_t hash() const { return _keys.empty() ? 0 : _keys[0]; } + uint32_t hash() const noexcept { return _keys.empty() ? 0 : _keys[0]; } bool operator==(const KeySet &rhs) const { return _keys == rhs._keys; } bool operator<(const KeySet &rhs) const { return _keys < rhs._keys; } bool contains(const KeySet &rhs) const; @@ -40,7 +40,7 @@ class BlobSet { public: class LidPosition { public: - LidPosition(uint32_t lid, uint32_t offset, uint32_t size) : _lid(lid), _offset(offset), _size(size) { } + LidPosition(uint32_t lid, uint32_t offset, uint32_t size) noexcept : _lid(lid), _offset(offset), _size(size) { } uint32_t lid() const { return _lid; } uint32_t offset() const { return _offset; } uint32_t size() const { return _size; } @@ -150,12 +150,12 @@ private: CompressedBlobSet readSet(const KeySet & keys); void removeKey(uint32_t key); private: - void locateAndInvalidateOtherSubsets(const vespalib::LockGuard & cacheGuard, const KeySet & keys); + void locateAndInvalidateOtherSubsets(const UniqueLock & cacheGuard, const KeySet & keys); using IdSet = vespalib::hash_set<uint64_t>; using Parent = vespalib::cache<CacheParams>; using LidUniqueKeySetId = vespalib::hash_map<uint32_t, uint64_t>; using IdKeySetMap = vespalib::hash_map<uint64_t, KeySet>; - IdSet findSetsContaining(const vespalib::LockGuard &, const KeySet & keys) const; + IdSet findSetsContaining(const UniqueLock &, const KeySet & keys) const; void onInsert(const K & key) override; void onRemove(const K & key) override; LidUniqueKeySetId _lid2Id; diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index 3517595d00a..cdf6220dfbc 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -204,7 +204,7 @@ seek_past(LidInfoWithLidV::const_iterator begin, LidInfoWithLidV::const_iterator } struct LidAndBuffer { - LidAndBuffer(uint32_t lid, uint32_t sz, vespalib::alloc::Alloc buf) : _lid(lid), _size(sz), _buf(std::move(buf)) {} + LidAndBuffer(uint32_t lid, uint32_t sz, vespalib::alloc::Alloc buf) noexcept : _lid(lid), _size(sz), _buf(std::move(buf)) {} uint32_t _lid; uint32_t _size; vespalib::alloc::Alloc _buf; @@ -220,7 +220,7 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB vespalib::hash_map<uint32_t, ChunkInfo> chunksOnFile; std::vector<LidAndBuffer> buffers; { - LockGuard guard(_lock); + vespalib::LockGuard guard(_lock); for (size_t i(0); i < count; i++) { const LidInfoWithLid & li = *(begin + i); uint32_t chunk = li.getChunkId(); @@ -260,7 +260,7 @@ WriteableFileChunk::read(uint32_t lid, SubChunkId chunkId, vespalib::DataBuffer { ChunkInfo chunkInfo; if (!frozen()) { - LockGuard guard(_lock); + vespalib::LockGuard guard(_lock); if ((chunkId >= _chunkInfo.size()) || !_chunkInfo[chunkId].valid()) { auto found = _chunkMap.find(chunkId); if (found != _chunkMap.end()) { @@ -282,7 +282,7 @@ WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum) { Chunk * active(nullptr); { - LockGuard guard(_lock); + vespalib::LockGuard guard(_lock); active = _chunkMap[chunkId].get(); } @@ -298,7 +298,7 @@ WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum) tmp->getBuf().moveFreeToData(padAfter); } { - LockGuard innerGuard(_lock); + vespalib::LockGuard innerGuard(_lock); setDiskFootprint(FileChunk::getDiskFootprint() + tmp->getBuf().getDataLen()); } enque(std::move(tmp)); @@ -396,11 +396,11 @@ WriteableFileChunk::fetchNextChain(ProcessedChunkMap & orderedChunks, const uint } ChunkMeta -WriteableFileChunk::computeChunkMeta(const LockGuard & guard, +WriteableFileChunk::computeChunkMeta(const vespalib::LockGuard & guard, const GenerationHandler::Guard & bucketizerGuard, size_t offset, const ProcessedChunk & tmp, const Chunk & active) { - (void) guard; + assert(guard.locks(_lock)); size_t dataLen = tmp.getBuf().getDataLen(); const ChunkMeta cmeta(offset, tmp.getPayLoad(), active.getLastSerial(), active.count()); assert((size_t(tmp.getBuf().getData())%_alignment) == 0); @@ -432,7 +432,7 @@ WriteableFileChunk::computeChunkMeta(ProcessedChunkQ & chunks, size_t startPos, cmetaV.reserve(chunks.size()); uint64_t lastSerial(_lastPersistedSerialNum); (void) lastSerial; - LockGuard guard(_lock); + vespalib::LockGuard guard(_lock); if (!_pendingChunks.empty()) { const PendingChunk & pc = *_pendingChunks.back(); @@ -541,7 +541,7 @@ WriteableFileChunk::fileWriter(const uint32_t firstChunkId) vespalib::system_time WriteableFileChunk::getModificationTime() const { - LockGuard guard(_lock); + vespalib::LockGuard guard(_lock); return _modificationTime; } @@ -595,7 +595,7 @@ size_t WriteableFileChunk::getMemoryFootprint() const { size_t sz(0); - LockGuard guard(_lock); + vespalib::LockGuard guard(_lock); for (const auto & it : _chunkMap) { sz += it.second->size(); } @@ -613,7 +613,7 @@ WriteableFileChunk::getMemoryMetaFootprint() const vespalib::MemoryUsage WriteableFileChunk::getMemoryUsage() const { - LockGuard guard(_lock); + vespalib::LockGuard guard(_lock); vespalib::MemoryUsage result; for (const auto &chunk : _chunkMap) { result.merge(chunk.second->getMemoryUsage()); @@ -843,7 +843,7 @@ WriteableFileChunk::updateCurrentDiskFootprint() { */ void WriteableFileChunk::flushPendingChunks(uint64_t serialNum) { - LockGuard flushGuard(_flushLock); + std::unique_lock flushGuard(_flushLock); if (frozen()) return; uint64_t datFileLen = _dataFile.getSize(); @@ -851,15 +851,14 @@ WriteableFileChunk::flushPendingChunks(uint64_t serialNum) { if (needFlushPendingChunks(serialNum, datFileLen)) { timeStamp = unconditionallyFlushPendingChunks(flushGuard, serialNum, datFileLen); } - LockGuard guard(_lock); + vespalib::LockGuard guard(_lock); _modificationTime = std::max(timeStamp, _modificationTime); } vespalib::system_time -WriteableFileChunk::unconditionallyFlushPendingChunks(const vespalib::LockGuard &flushGuard, uint64_t serialNum, uint64_t datFileLen) +WriteableFileChunk::unconditionallyFlushPendingChunks(const LockGuard &flushGuard, uint64_t serialNum, uint64_t datFileLen) { - (void) flushGuard; - assert(flushGuard.locks(_flushLock)); + assert((flushGuard.mutex() == &_flushLock) && flushGuard.owns_lock()); if ( ! _dataFile.Sync()) { throw SummaryException("Failed fsync of dat file", _dataFile, VESPA_STRLOC); } diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 65dd822ac1f..cdbbd51ec77 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -4,6 +4,7 @@ #include "filechunk.h" #include <vespa/vespalib/util/executor.h> +#include <vespa/vespalib/util/sync.h> #include <vespa/searchlib/transactionlog/syncproxy.h> #include <vespa/fastos/file.h> #include <map> @@ -90,7 +91,7 @@ private: void writeDataHeader(const common::FileHeaderContext &fileHeaderContext); bool needFlushPendingChunks(uint64_t serialNum, uint64_t datFileLen); bool needFlushPendingChunks(const vespalib::MonitorGuard & guard, uint64_t serialNum, uint64_t datFileLen); - vespalib::system_time unconditionallyFlushPendingChunks(const vespalib::LockGuard & flushGuard, uint64_t serialNum, uint64_t datFileLen); + vespalib::system_time unconditionallyFlushPendingChunks(const LockGuard & flushGuard, uint64_t serialNum, uint64_t datFileLen); static void insertChunks(ProcessedChunkMap & orderedChunks, ProcessedChunkQ & newChunks, const uint32_t nextChunkId); static ProcessedChunkQ fetchNextChain(ProcessedChunkMap & orderedChunks, const uint32_t firstChunkId); ChunkMeta computeChunkMeta(const vespalib::LockGuard & guard, @@ -108,8 +109,8 @@ private: bool _frozen; // Lock order is _writeLock, _flushLock, _lock vespalib::Monitor _lock; - vespalib::Lock _writeLock; - vespalib::Lock _flushLock; + std::mutex _writeLock; + std::mutex _flushLock; FastOS_File _dataFile; using ChunkMap = std::map<uint32_t, Chunk::UP>; ChunkMap _chunkMap; diff --git a/searchlib/src/vespa/searchlib/engine/docsumreply.h b/searchlib/src/vespa/searchlib/engine/docsumreply.h index 37431030253..0150b1eda3d 100644 --- a/searchlib/src/vespa/searchlib/engine/docsumreply.h +++ b/searchlib/src/vespa/searchlib/engine/docsumreply.h @@ -22,9 +22,9 @@ struct DocsumReply document::GlobalId gid; Blob data; - Docsum() : docid(0), gid(), data(0) {} - Docsum(document::GlobalId gid_) : docid(0), gid(gid_), data(0) { } - Docsum(document::GlobalId gid_, const char *buf, uint32_t len) : docid(0), gid(gid_), data(len) { + Docsum() noexcept : docid(0), gid(), data(0) {} + Docsum(document::GlobalId gid_) noexcept : docid(0), gid(gid_), data(0) { } + Docsum(document::GlobalId gid_, const char *buf, uint32_t len) noexcept : docid(0), gid(gid_), data(len) { memcpy(data.str(), buf, len); } Docsum & setData(const char *buf, uint32_t len) { diff --git a/searchlib/src/vespa/searchlib/engine/docsumrequest.h b/searchlib/src/vespa/searchlib/engine/docsumrequest.h index 8aa8d036b73..8fe5aa6f465 100644 --- a/searchlib/src/vespa/searchlib/engine/docsumrequest.h +++ b/searchlib/src/vespa/searchlib/engine/docsumrequest.h @@ -23,8 +23,8 @@ public: class Hit { public: - Hit() : gid(), docid(0), path(0) {} - Hit(const document::GlobalId & gid_) : gid(gid_), docid(0), path(0) {} + Hit() noexcept : gid(), docid(0), path(0) {} + Hit(const document::GlobalId & gid_) noexcept : gid(gid_), docid(0), path(0) {} document::GlobalId gid; mutable uint32_t docid; // converted in backend diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h index d9468216a90..531f94d2b6f 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.h +++ b/searchlib/src/vespa/searchlib/engine/searchreply.h @@ -18,7 +18,7 @@ public: class Hit { public: - Hit() : gid(), metric(0), path(0), _distributionKey(0) {} + Hit() noexcept : gid(), metric(0), path(0), _distributionKey(0) {} void setDistributionKey(uint32_t key) { _distributionKey = key; } uint32_t getDistributionKey() const { return _distributionKey; } document::GlobalId gid; @@ -30,9 +30,9 @@ public: class Coverage { public: - Coverage() : Coverage(0) { } - Coverage(uint64_t active) : Coverage(active, active) { } - Coverage(uint64_t active, uint64_t covered) + Coverage() noexcept : Coverage(0) { } + Coverage(uint64_t active) noexcept : Coverage(active, active) { } + Coverage(uint64_t active, uint64_t covered) noexcept : _covered(covered), _active(active), _soonActive(active), _degradeReason(0), _nodesQueried(1), _nodesReplied(1) { } diff --git a/searchlib/src/vespa/searchlib/features/array_parser.h b/searchlib/src/vespa/searchlib/features/array_parser.h index 8bf6e9ca365..7de766efddc 100644 --- a/searchlib/src/vespa/searchlib/features/array_parser.h +++ b/searchlib/src/vespa/searchlib/features/array_parser.h @@ -29,7 +29,7 @@ public: class ValueAndIndex { public: typedef T ValueType; - ValueAndIndex(T value, uint32_t index) : _value(value), _index(index) { } + ValueAndIndex(T value, uint32_t index) noexcept : _value(value), _index(index) { } T getValue() const { return _value; } uint32_t getIndex() const { return _index; } bool operator < (const ValueAndIndex & b) const { return _index < b._index; } diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.h b/searchlib/src/vespa/searchlib/features/bm25_feature.h index 0afd14e7ac8..72dcb7e2ef7 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.h +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.h @@ -14,7 +14,7 @@ private: fef::TermFieldHandle handle; const fef::TermFieldMatchData* tfmd; double idf_mul_k1_plus_one; - QueryTerm(fef::TermFieldHandle handle_, double inverse_doc_freq, double k1_param) + QueryTerm(fef::TermFieldHandle handle_, double inverse_doc_freq, double k1_param) noexcept : handle(handle_), tfmd(nullptr), idf_mul_k1_plus_one(inverse_doc_freq * (k1_param + 1)) diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h index e4dbde1248a..a4699b17457 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h @@ -289,7 +289,7 @@ private: struct SegmentData { SegmentData() : segment(), valid(false) {} - SegmentData(SegmentStart::SP ss, bool v = false) : segment(std::move(ss)), valid(v) {} + SegmentData(SegmentStart::SP ss, bool v = false) noexcept : segment(std::move(ss)), valid(v) {} SegmentStart::SP segment; bool valid; }; diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index 0bc85a63ceb..dd1e774607f 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -23,7 +23,7 @@ struct Override BlueprintResolver::FeatureRef ref; feature_t value; - Override(const BlueprintResolver::FeatureRef &r, feature_t v) + Override(const BlueprintResolver::FeatureRef &r, feature_t v) noexcept : ref(r), value(v) {} bool operator<(const Override &rhs) const { diff --git a/searchlib/src/vespa/searchlib/fef/tablemanager.cpp b/searchlib/src/vespa/searchlib/fef/tablemanager.cpp index 9503e34cbaa..1aa7d4a14e9 100644 --- a/searchlib/src/vespa/searchlib/fef/tablemanager.cpp +++ b/searchlib/src/vespa/searchlib/fef/tablemanager.cpp @@ -2,36 +2,29 @@ #include "tablemanager.h" -namespace search { -namespace fef { +namespace search::fef { -TableManager::TableManager() : - _factories(), - _cache(), - _lock() -{ -} +TableManager::TableManager() = default; -TableManager::~TableManager() {} +TableManager::~TableManager() = default; const Table * TableManager::getTable(const vespalib::string & name) const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); TableCache::const_iterator itr = _cache.find(name); if (itr != _cache.end()) { return itr->second.get(); } for (size_t i = 0; i < _factories.size(); ++i) { Table::SP table = _factories[i]->createTable(name); - if (table.get() != NULL) { + if (table) { _cache.insert(std::make_pair(name, table)); return table.get(); } } - _cache.insert(std::make_pair(name, Table::SP(NULL))); - return NULL; + _cache.insert(std::make_pair(name, Table::SP())); + return nullptr; } -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/tablemanager.h b/searchlib/src/vespa/searchlib/fef/tablemanager.h index 4efd8d4aeff..4b83cb7c7a0 100644 --- a/searchlib/src/vespa/searchlib/fef/tablemanager.h +++ b/searchlib/src/vespa/searchlib/fef/tablemanager.h @@ -4,12 +4,11 @@ #include "itablefactory.h" #include "itablemanager.h" -#include <vespa/vespalib/util/sync.h> #include <map> #include <vector> +#include <mutex> -namespace search { -namespace fef { +namespace search::fef { /** * This class manages a set of tables and contains an ordered list of table factories used to create tables, @@ -24,11 +23,11 @@ private: typedef std::map<vespalib::string, Table::SP> TableCache; std::vector<ITableFactory::SP> _factories; mutable TableCache _cache; - vespalib::Lock _lock; + mutable std::mutex _lock; public: TableManager(); - ~TableManager(); + ~TableManager() override; /** * Adds a table factory to this manager. @@ -46,6 +45,4 @@ public: const Table * getTable(const vespalib::string & name) const override; }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp index 973e11fc0d2..11a5dd80b4c 100644 --- a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp +++ b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp @@ -24,7 +24,7 @@ TermMatchDataMerger::TermMatchDataMerger(const Inputs &allinputs, } } -TermMatchDataMerger::~TermMatchDataMerger() {} +TermMatchDataMerger::~TermMatchDataMerger() = default; void TermMatchDataMerger::merge(uint32_t docid) diff --git a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.h b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.h index e0addc374b2..fb129b792b8 100644 --- a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.h +++ b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.h @@ -6,8 +6,7 @@ #include "termfieldmatchdata.h" #include <vector> -namespace search { -namespace fef { +namespace search::fef { class TermMatchDataMerger { @@ -16,8 +15,8 @@ public: const TermFieldMatchData *matchData; double exactness; - Input() : matchData(NULL), exactness(0.0) {} - Input(const TermFieldMatchData *arg_matchData, double arg_exactness) + Input() : matchData(nullptr), exactness(0.0) {} + Input(const TermFieldMatchData *arg_matchData, double arg_exactness) noexcept : matchData(arg_matchData), exactness(arg_exactness) {} }; @@ -27,21 +26,16 @@ private: const TermFieldMatchDataArray _output; std::vector<TermFieldMatchDataPosition> _scratch; - TermMatchDataMerger(const TermMatchDataMerger &); - TermMatchDataMerger &operator=(const TermMatchDataMerger &); - - void merge(uint32_t docid, - const Inputs &in, - TermFieldMatchData &out); + void merge(uint32_t docid, const Inputs &in, TermFieldMatchData &out); public: + TermMatchDataMerger(const TermMatchDataMerger &) = delete; + TermMatchDataMerger &operator=(const TermMatchDataMerger &) = delete; - TermMatchDataMerger(const Inputs &allinputs, - const TermFieldMatchDataArray &outputs); + TermMatchDataMerger(const Inputs &allinputs, const TermFieldMatchDataArray &outputs); ~TermMatchDataMerger(); void merge(uint32_t docid); }; -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/grouping/hyperloglog.h b/searchlib/src/vespa/searchlib/grouping/hyperloglog.h index 931b832c76d..2206f4ccbe1 100644 --- a/searchlib/src/vespa/searchlib/grouping/hyperloglog.h +++ b/searchlib/src/vespa/searchlib/grouping/hyperloglog.h @@ -28,7 +28,7 @@ template <int BucketBits = 10, typename HashT = uint32_t> class ExchangerSketch : public SparseSketch<BucketBits, HashT> { typename Sketch<BucketBits, HashT>::UP &_sketch_ptr; - virtual int aggregate(HashT hash) override { + int aggregate(HashT hash) override { if (this->getSize() > SPARSE_SKETCH_LIMIT) { NormalSketch<BucketBits, HashT> *normal_sketch = new NormalSketch<BucketBits, HashT>; diff --git a/searchlib/src/vespa/searchlib/grouping/sketch.h b/searchlib/src/vespa/searchlib/grouping/sketch.h index 317c1bfef9d..c105b480a3d 100644 --- a/searchlib/src/vespa/searchlib/grouping/sketch.h +++ b/searchlib/src/vespa/searchlib/grouping/sketch.h @@ -70,21 +70,21 @@ struct SparseSketch : Sketch<BucketBits, HashT> { enum { classId = IDENTIFIABLE_CLASSID_NS(search, SparseSketch) }; struct IdentityHash { - size_t operator()(HashT hash) const { return hash; } + size_t operator()(HashT hash) const noexcept { return hash; } }; std::unordered_set<HashT, IdentityHash> hash_set; size_t getSize() const { return hash_set.size(); } - virtual int aggregate(HashT hash) override { + int aggregate(HashT hash) override { return hash_set.insert(hash).second ? 1 : 0; } - virtual uint32_t getClassId() const override { return classId; } - virtual void serialize(vespalib::Serializer &os) const override; - virtual void deserialize(vespalib::Deserializer &is) override; + uint32_t getClassId() const override { return classId; } + void serialize(vespalib::Serializer &os) const override; + void deserialize(vespalib::Deserializer &is) override; - virtual bool operator==(const SketchType &other) const override { + bool operator==(const SketchType &other) const override { const SparseSketch<BucketBits, HashT> *other_sparse = dynamic_cast<const SparseSketch<BucketBits, HashT> *>(&other); if (!other_sparse) { @@ -101,7 +101,7 @@ struct SparseSketch : Sketch<BucketBits, HashT> { return true; } - virtual void print(std::ostream &out) const override { + void print(std::ostream &out) const override { out << " (" << hash_set.size() << " elements)"; for (auto hash : hash_set) { out << " 0x" << std::hex; diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.h b/searchlib/src/vespa/searchlib/index/docidandfeatures.h index 5372d5ef3aa..6ee80721038 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.h +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.h @@ -26,14 +26,14 @@ private: uint32_t _elementLen; public: - WordDocElementFeatures() + WordDocElementFeatures() noexcept : _elementId(0u), _numOccs(0u), _weight(1), _elementLen(SEARCHLIB_FEF_UNKNOWN_FIELD_LENGTH) {} - WordDocElementFeatures(uint32_t elementId) + WordDocElementFeatures(uint32_t elementId) noexcept : _elementId(elementId), _numOccs(0u), _weight(1), @@ -42,7 +42,7 @@ public: WordDocElementFeatures(uint32_t elementId, uint32_t weight, - uint32_t elementLen) + uint32_t elementLen) noexcept : _elementId(elementId), _numOccs(0u), _weight(weight), @@ -71,11 +71,11 @@ private: uint32_t _wordPos; public: - WordDocElementWordPosFeatures() + WordDocElementWordPosFeatures() noexcept : _wordPos(0u) {} - WordDocElementWordPosFeatures(uint32_t wordPos) + WordDocElementWordPosFeatures(uint32_t wordPos) noexcept : _wordPos(wordPos) {} @@ -101,17 +101,17 @@ protected: std::vector<WordDocElementWordPosFeatures> _word_positions; // Raw data (file format specific, packed) - RawData _blob; // Feature data for (word, docid) pair + RawData _blob; // Feature data for (word, docid) pair uint32_t _bit_offset; // Offset of feature start ([0..63]) uint32_t _bit_length; // Length of features - bool _has_raw_data; + bool _has_raw_data; public: DocIdAndFeatures(); DocIdAndFeatures(const DocIdAndFeatures &); DocIdAndFeatures & operator = (const DocIdAndFeatures &); - DocIdAndFeatures(DocIdAndFeatures &&) = default; - DocIdAndFeatures & operator = (DocIdAndFeatures &&) = default; + DocIdAndFeatures(DocIdAndFeatures &&) noexcept = default; + DocIdAndFeatures & operator = (DocIdAndFeatures &&) noexcept = default; ~DocIdAndFeatures(); void clear_features() { diff --git a/searchlib/src/vespa/searchlib/index/schemautil.cpp b/searchlib/src/vespa/searchlib/index/schemautil.cpp index 1fce4a1fe99..57a90892d4f 100644 --- a/searchlib/src/vespa/searchlib/index/schemautil.cpp +++ b/searchlib/src/vespa/searchlib/index/schemautil.cpp @@ -3,6 +3,7 @@ #include "schemautil.h" #include <set> #include <fstream> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".index.schemautil"); diff --git a/searchlib/src/vespa/searchlib/index/uri_field.cpp b/searchlib/src/vespa/searchlib/index/uri_field.cpp index 30b78c24410..af0881dd6b4 100644 --- a/searchlib/src/vespa/searchlib/index/uri_field.cpp +++ b/searchlib/src/vespa/searchlib/index/uri_field.cpp @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "uri_field.h" +#include <cassert> namespace search::index { diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h index 36d2286cfb1..717f21528bb 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h @@ -21,11 +21,11 @@ private: struct WordFieldDocTuple { vespalib::datastore::EntryRef _wordRef; uint32_t _docId; - WordFieldDocTuple() : + WordFieldDocTuple() noexcept : _wordRef(0), _docId(0) { } - WordFieldDocTuple(vespalib::datastore::EntryRef wordRef, uint32_t docId) : + WordFieldDocTuple(vespalib::datastore::EntryRef wordRef, uint32_t docId) noexcept : _wordRef(wordRef), _docId(docId) { } @@ -44,12 +44,12 @@ private: CompactWordsStore _store; CompactWordsStore::Builder::UP _builder; - std::vector<WordFieldDocTuple> _wordFieldDocTuples; + std::vector<WordFieldDocTuple> _wordFieldDocTuples; const WordStore &_wordStore; public: FieldIndexRemover(const WordStore &wordStore); - ~FieldIndexRemover(); + ~FieldIndexRemover() override; void remove(uint32_t docId, IFieldIndexRemoveListener &inverter); CompactWordsStore &getStore() { return _store; } const CompactWordsStore &getStore() const { return _store; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 78a1cf6c171..3a18d0c2f8c 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -39,7 +39,7 @@ public: static constexpr uint32_t _elemRemoved = std::numeric_limits<uint32_t>::max(); - PosInfo() + PosInfo() noexcept : _wordNum(0), _docId(0), _elemId(0), @@ -51,7 +51,7 @@ public: PosInfo(uint32_t wordRef, uint32_t docId, uint32_t elemId, - uint32_t wordPos, uint32_t elemRef) + uint32_t wordPos, uint32_t elemRef) noexcept : _wordNum(wordRef), _docId(docId), _elemId(elemId), @@ -60,8 +60,7 @@ public: { } - PosInfo(uint32_t wordRef, - uint32_t docId) + PosInfo(uint32_t wordRef, uint32_t docId) noexcept : _wordNum(wordRef), _docId(docId), _elemId(_elemRemoved), diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 34c5d65ab23..789b7fcf452 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -18,7 +18,6 @@ LOG_SETUP(".searchlib.memoryindex.memory_index"); using document::ArrayFieldValue; using document::WeightedSetFieldValue; -using vespalib::LockGuard; namespace search { @@ -215,7 +214,7 @@ MemoryIndex::getNumWords() const { void MemoryIndex::pruneRemovedFields(const Schema &schema) { - LockGuard lock(_lock); + std::lock_guard lock(_lock); if (_prunedSchema.get() == nullptr) { auto newSchema = Schema::intersect(_schema, schema); if (_schema == *newSchema) { @@ -241,7 +240,7 @@ MemoryIndex::pruneRemovedFields(const Schema &schema) Schema::SP MemoryIndex::getPrunedSchema() const { - LockGuard lock(_lock); + std::lock_guard lock(_lock); return _prunedSchema; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index a9f153f7dd8..56eb4ad7a66 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -48,14 +48,14 @@ private: std::unique_ptr<DocumentInverter> _inverter0; std::unique_ptr<DocumentInverter> _inverter1; DocumentInverter *_inverter; - bool _frozen; - uint32_t _maxDocId; - uint32_t _numDocs; - vespalib::Lock _lock; - std::vector<bool> _hiddenFields; - index::Schema::SP _prunedSchema; + bool _frozen; + uint32_t _maxDocId; + uint32_t _numDocs; + mutable std::mutex _lock; + std::vector<bool> _hiddenFields; + index::Schema::SP _prunedSchema; vespalib::hash_set<uint32_t> _indexedDocs; // documents in memory index - const uint64_t _staticMemoryFootprint; + const uint64_t _staticMemoryFootprint; MemoryIndex(const MemoryIndex &) = delete; MemoryIndex(MemoryIndex &&) = delete; diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index 1820fb0e969..6039a86580c 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -225,7 +225,7 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_PREDICATE_QUERY: try { _curr_index_name = read_stringref(p); - _predicate_query_term.reset(new PredicateQueryTerm); + _predicate_query_term = std::make_unique<PredicateQueryTerm>(); size_t count = readCompressedPositiveInt(p); for (size_t i = 0; i < count; ++i) { diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp index e8257d28c63..dba6ebfb117 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp @@ -4,6 +4,7 @@ #include <vespa/document/predicate/predicate.h> #include <algorithm> #include <cmath> +#include <cassert> using document::Predicate; using std::map; diff --git a/searchlib/src/vespa/searchlib/query/streaming/hit.h b/searchlib/src/vespa/searchlib/query/streaming/hit.h index 64b71a70df9..fabaa3ad50c 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/hit.h +++ b/searchlib/src/vespa/searchlib/query/streaming/hit.h @@ -9,7 +9,7 @@ namespace search::streaming { class Hit { public: - Hit(uint32_t pos_, uint32_t context_, uint32_t elemId_, int32_t weight_) + Hit(uint32_t pos_, uint32_t context_, uint32_t elemId_, int32_t weight_) noexcept : _position(pos_ | (context_<<24)), _elemId(elemId_), _weight(weight_) diff --git a/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h b/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h index 0a92546e414..8602eb1ac57 100644 --- a/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h +++ b/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h @@ -22,7 +22,7 @@ class PredicateQueryTerm { public: Entry(const vespalib::string &key, const ValueType &value, - uint64_t sub_query_bitmap = ALL_SUB_QUERIES) + uint64_t sub_query_bitmap = ALL_SUB_QUERIES) noexcept : _key(key), _value(value), _sub_query_bitmap(sub_query_bitmap) {} vespalib::string getKey() const { return _key; } @@ -41,13 +41,7 @@ class PredicateQueryTerm { public: typedef std::unique_ptr<PredicateQueryTerm> UP; - PredicateQueryTerm() : _features(), _range_features() {} - - PredicateQueryTerm(const std::vector<Entry<vespalib::string>> &features, - const std::vector<Entry<uint64_t>> &range_features) - : _features(features), - _range_features(range_features) { - } + PredicateQueryTerm() noexcept : _features(), _range_features() {} void addFeature(const vespalib::string &key, const vespalib::string &value, uint64_t sub_query_bitmask = ALL_SUB_QUERIES) { diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index 9289df7cbe9..600249c3e1e 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -153,7 +153,7 @@ private: void visit(PredicateQuery &node) override { replicate(node, _builder.addPredicateQuery( - PredicateQueryTerm::UP(new PredicateQueryTerm(*node.getTerm())), + std::make_unique<PredicateQueryTerm>(*node.getTerm()), node.getView(), node.getId(), node.getWeight())); } diff --git a/searchlib/src/vespa/searchlib/queryeval/global_filter.h b/searchlib/src/vespa/searchlib/queryeval/global_filter.h index b30162affa7..3e57ded4898 100644 --- a/searchlib/src/vespa/searchlib/queryeval/global_filter.h +++ b/searchlib/src/vespa/searchlib/queryeval/global_filter.h @@ -21,15 +21,15 @@ private: struct ctor_tag {}; std::unique_ptr<search::BitVector> bit_vector; +public: GlobalFilter(const GlobalFilter &) = delete; GlobalFilter(GlobalFilter &&) = delete; -public: - GlobalFilter(ctor_tag, std::unique_ptr<search::BitVector> bit_vector_in) + GlobalFilter(ctor_tag, std::unique_ptr<search::BitVector> bit_vector_in) noexcept : bit_vector(std::move(bit_vector_in)) {} - GlobalFilter(ctor_tag) : bit_vector() {} + GlobalFilter(ctor_tag) noexcept : bit_vector() {} ~GlobalFilter() {} diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h index f209e0f7fd8..b9812036637 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h @@ -29,7 +29,7 @@ public: SearchIterator *search; uint32_t sourceId; Child() : search(nullptr), sourceId(0) { } - Child(SearchIterator *s, uint32_t id) : search(s), sourceId(id) {} + Child(SearchIterator *s, uint32_t id) noexcept : search(s), sourceId(id) {} }; typedef std::vector<Child> Children; diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h index 071d6d99470..937e7a089df 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h @@ -41,11 +41,12 @@ struct Term { uint32_t estHits; fef::TermFieldMatchData *matchData; score_t maxScore = 0.0; // <- only used by rise wand test - Term(SearchIterator *s, int32_t w, uint32_t e, fef::TermFieldMatchData *tfmd) - : search(s), weight(w), estHits(e), matchData(tfmd) {} - Term() : Term(nullptr, 0, 0, nullptr){} - Term(SearchIterator *s, int32_t w, uint32_t e) : Term(s, w, e, nullptr) {} - Term(SearchIterator::UP s, int32_t w, uint32_t e) : Term(s.release(), w, e, nullptr) {} + Term(SearchIterator *s, int32_t w, uint32_t e, fef::TermFieldMatchData *tfmd) noexcept + : search(s), weight(w), estHits(e), matchData(tfmd) + {} + Term() noexcept : Term(nullptr, 0, 0, nullptr){} + Term(SearchIterator *s, int32_t w, uint32_t e) noexcept : Term(s, w, e, nullptr) {} + Term(SearchIterator::UP s, int32_t w, uint32_t e) noexcept : Term(s.release(), w, e, nullptr) {} }; //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.cpp index dc84eb45b8a..28c229596b0 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.cpp @@ -12,7 +12,7 @@ SharedWeakAndPriorityQueue::SharedWeakAndPriorityQueue(uint32_t scoresToTrack) : _bestScores.reserve(scoresToTrack); } -SharedWeakAndPriorityQueue::~SharedWeakAndPriorityQueue() { } +SharedWeakAndPriorityQueue::~SharedWeakAndPriorityQueue() = default; void SharedWeakAndPriorityQueue::adjust(score_t *begin, score_t *end) @@ -20,7 +20,7 @@ SharedWeakAndPriorityQueue::adjust(score_t *begin, score_t *end) if (getScoresToTrack() == 0) { return; } - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); for (score_t *itr = begin; itr != end; ++itr) { score_t score = *itr; if (!is_full()) { diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h index d6105d7e6f2..28c317de3ee 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h @@ -3,7 +3,7 @@ #include "wand_parts.h" #include <vespa/vespalib/util/priority_queue.h> -#include <vespa/vespalib/util/sync.h> +#include <mutex> namespace search::queryeval { @@ -51,7 +51,7 @@ class SharedWeakAndPriorityQueue : public WeakAndHeap private: typedef vespalib::PriorityQueue<score_t> Scores; Scores _bestScores; - vespalib::Lock _lock; + std::mutex _lock; bool is_full() const { return (_bestScores.size() >= getScoresToTrack()); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 9def5a7b0a8..6488b525b7c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -40,7 +40,7 @@ struct PairDist { uint32_t id_first; uint32_t id_second; double distance; - PairDist(uint32_t i1, uint32_t i2, double d) + PairDist(uint32_t i1, uint32_t i2, double d) noexcept : id_first(i1), id_second(i2), distance(d) {} }; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h index 99266505780..d206b055071 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h @@ -16,9 +16,9 @@ struct HnswCandidate { uint32_t docid; HnswGraph::NodeRef node_ref; double distance; - HnswCandidate(uint32_t docid_in, double distance_in) + HnswCandidate(uint32_t docid_in, double distance_in) noexcept : docid(docid_in), node_ref(), distance(distance_in) {} - HnswCandidate(uint32_t docid_in, HnswGraph::NodeRef node_ref_in, double distance_in) + HnswCandidate(uint32_t docid_in, HnswGraph::NodeRef node_ref_in, double distance_in) noexcept : docid(docid_in), node_ref(node_ref_in), distance(distance_in) {} }; diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index 74f14cea21b..c14da0d058f 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -29,12 +29,12 @@ public: struct Neighbor { uint32_t docid; double distance; - Neighbor(uint32_t id, double dist) + Neighbor(uint32_t id, double dist) noexcept : docid(id), distance(dist) {} - Neighbor() : docid(0), distance(0.0) {} + Neighbor() noexcept : docid(0), distance(0.0) {} }; - virtual ~NearestNeighborIndex() {} + virtual ~NearestNeighborIndex() = default; virtual void add_document(uint32_t docid) = 0; /** diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h index e2fcbf444a2..0a04e1e79fd 100644 --- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h +++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h @@ -23,7 +23,7 @@ namespace search { struct MockDocumentMetaStoreContext : public IDocumentMetaStoreContext { mutable size_t get_read_guard_cnt; - MockDocumentMetaStoreContext() : get_read_guard_cnt(0) {} + MockDocumentMetaStoreContext() noexcept : get_read_guard_cnt(0) {} IReadGuard::UP getReadGuard() const override; }; diff --git a/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp b/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp index 8678d88b43c..40db92cbe78 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp @@ -123,7 +123,7 @@ void Session::clear() { if (_sessionId > 0) { - LockGuard guard(_tlc._lock); + std::lock_guard guard(_tlc._lock); _tlc._sessions.erase(SessionKey(_domain, _sessionId)); } _sessionId = 0; @@ -145,7 +145,7 @@ Session::init(FRT_RPCRequest *req) _sessionId = retval; SessionKey key(_domain, _sessionId); { - LockGuard guard(_tlc._lock); + std::lock_guard guard(_tlc._lock); _tlc._sessions[key] = this; } retval = run(); diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 126a7afed4d..454293dfc84 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -19,7 +19,6 @@ LOG_SETUP(".transactionlog.domain"); using vespalib::string; using vespalib::make_string_short::fmt; -using vespalib::LockGuard; using vespalib::makeTask; using vespalib::makeClosure; using vespalib::makeLambdaTask; @@ -104,7 +103,7 @@ Domain::addPart(SerialNum partId, bool isLastPart) { dp->erase(dp->range().to() + 1); } else { { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _parts[partId] = dp; } if (! isLastPart) { @@ -123,7 +122,7 @@ Domain::~Domain() { DomainInfo Domain::getDomainInfo() const { - LockGuard guard(_lock); + UniqueLock guard(_lock); DomainInfo info(SerialNumRange(begin(guard), end(guard)), size(guard), byteSize(guard), _maxSessionRunTime); for (const auto &entry: _parts) { const DomainPart &part = *entry.second; @@ -135,14 +134,18 @@ Domain::getDomainInfo() const SerialNum Domain::begin() const { - return begin(LockGuard(_lock)); + return begin(UniqueLock(_lock)); } +void +Domain::verifyLock(const UniqueLock & guard) const { + assert(guard.mutex() == &_lock); + assert(guard.owns_lock()); +} SerialNum -Domain::begin(const LockGuard & guard) const +Domain::begin(const UniqueLock & guard) const { - (void) guard; - assert(guard.locks(_lock)); + verifyLock(guard); SerialNum s(0); if ( ! _parts.empty() ) { s = _parts.cbegin()->second->range().from(); @@ -153,14 +156,13 @@ Domain::begin(const LockGuard & guard) const SerialNum Domain::end() const { - return end(LockGuard(_lock)); + return end(UniqueLock(_lock)); } SerialNum -Domain::end(const LockGuard & guard) const +Domain::end(const UniqueLock & guard) const { - (void) guard; - assert(guard.locks(_lock)); + verifyLock(guard); SerialNum s(0); if ( ! _parts.empty() ) { s = _parts.crbegin()->second->range().to(); @@ -171,14 +173,13 @@ Domain::end(const LockGuard & guard) const size_t Domain::byteSize() const { - return byteSize(LockGuard(_lock)); + return byteSize(UniqueLock(_lock)); } size_t -Domain::byteSize(const LockGuard & guard) const +Domain::byteSize(const UniqueLock & guard) const { - (void) guard; - assert(guard.locks(_lock)); + verifyLock(guard); size_t size = 0; for (const auto &entry : _parts) { const DomainPart &part = *entry.second; @@ -191,7 +192,7 @@ SerialNum Domain::getSynced() const { SerialNum s(0); - LockGuard guard(_lock); + UniqueLock guard(_lock); if (_parts.empty()) { return s; } @@ -228,7 +229,7 @@ Domain::triggerSyncNow() DomainPart::SP Domain::findPart(SerialNum s) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); DomainPartList::iterator it(_parts.upper_bound(s)); if (!_parts.empty() && it != _parts.begin()) { DomainPartList::iterator prev(it); @@ -246,14 +247,13 @@ Domain::findPart(SerialNum s) uint64_t Domain::size() const { - return size(LockGuard(_lock)); + return size(UniqueLock(_lock)); } uint64_t -Domain::size(const LockGuard & guard) const +Domain::size(const UniqueLock & guard) const { - (void) guard; - assert(guard.locks(_lock)); + verifyLock(guard); uint64_t sz(0); for (const auto & part : _parts) { sz += part.second->size(); @@ -265,7 +265,7 @@ SerialNum Domain::findOldestActiveVisit() const { SerialNum oldestActive(std::numeric_limits<SerialNum>::max()); - LockGuard guard(_sessionLock); + std::lock_guard guard(_sessionLock); for (const auto & pair : _sessions) { Session * session(pair.second.get()); if (!session->inSync()) { @@ -281,7 +281,7 @@ Domain::cleanSessions() if ( _sessions.empty()) { return; } - LockGuard guard(_sessionLock); + std::lock_guard guard(_sessionLock); for (SessionList::iterator it(_sessions.begin()), mt(_sessions.end()); it != mt; ) { Session * session(it->second.get()); if (session->inSync()) { @@ -318,7 +318,7 @@ Domain::optionallyRotateFile(SerialNum serialNum) { dp = std::make_shared<DomainPart>(_name, dir(), serialNum, _config.getEncoding(), _config.getCompressionlevel(), _fileHeaderContext, false); { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _parts[serialNum] = dp; assert(_parts.rbegin()->first == serialNum); } @@ -409,7 +409,7 @@ Domain::erase(SerialNum to) for (DomainPartList::iterator it(_parts.begin()); (_parts.size() > 1) && (it->second.get()->range().to() < to); it = _parts.begin()) { DomainPart::SP dp(it->second); { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _parts.erase(it); } retval = retval && dp->erase(to); @@ -429,7 +429,7 @@ Domain::visit(const Domain::SP & domain, SerialNum from, SerialNum to, std::uniq SerialNumRange range(from, to); auto session = std::make_shared<Session>(_sessionId++, range, domain, std::move(dest)); int id = session->id(); - LockGuard guard(_sessionLock); + std::lock_guard guard(_sessionLock); _sessions[id] = std::move(session); return id; } @@ -438,7 +438,7 @@ int Domain::startSession(int sessionId) { int retval(-1); - LockGuard guard(_sessionLock); + std::lock_guard guard(_sessionLock); SessionList::iterator found = _sessions.find(sessionId); if (found != _sessions.end()) { found->second->setStartTime(std::chrono::steady_clock::now()); @@ -458,7 +458,7 @@ Domain::closeSession(int sessionId) int retval(-1); DurationSeconds sessionRunTime(0); { - LockGuard guard(_sessionLock); + std::lock_guard guard(_sessionLock); SessionList::iterator found = _sessions.find(sessionId); if (found != _sessions.end()) { sessionRunTime = (std::chrono::steady_clock::now() - found->second->getStartTime()); @@ -467,7 +467,7 @@ Domain::closeSession(int sessionId) } while (retval == 1) { std::this_thread::sleep_for(10ms); - LockGuard guard(_sessionLock); + std::lock_guard guard(_sessionLock); SessionList::iterator found = _sessions.find(sessionId); if (found != _sessions.end()) { if ( ! found->second->isVisitRunning()) { @@ -479,7 +479,7 @@ Domain::closeSession(int sessionId) } } { - LockGuard guard(_lock); + std::lock_guard guard(_lock); if (sessionRunTime > _maxSessionRunTime) { _maxSessionRunTime = sessionRunTime; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index e41ad930840..a883156f32c 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,16 +56,18 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: + using UniqueLock = std::unique_lock<std::mutex>; + void verifyLock(const UniqueLock & guard) const; void commitIfFull(const vespalib::MonitorGuard & guard); void commitAndTransferResponses(const vespalib::MonitorGuard & guard); std::unique_ptr<CommitChunk> grabCurrentChunk(const vespalib::MonitorGuard & guard); void commitChunk(std::unique_ptr<CommitChunk> chunk, const vespalib::MonitorGuard & chunkOrderGuard); void doCommit(std::unique_ptr<CommitChunk> chunk); - SerialNum begin(const vespalib::LockGuard & guard) const; - SerialNum end(const vespalib::LockGuard & guard) const; - size_t byteSize(const vespalib::LockGuard & guard) const; - uint64_t size(const vespalib::LockGuard & guard) const; + SerialNum begin(const UniqueLock & guard) const; + SerialNum end(const UniqueLock & guard) const; + size_t byteSize(const UniqueLock & guard) const; + uint64_t size(const UniqueLock & guard) const; void cleanSessions(); vespalib::string dir() const { return getDir(_baseDir, _name); } void addPart(SerialNum partId, bool isLastPart); @@ -89,9 +91,9 @@ private: bool _pendingSync; vespalib::string _name; DomainPartList _parts; - vespalib::Lock _lock; + mutable std::mutex _lock; vespalib::Monitor _currentChunkMonitor; - vespalib::Lock _sessionLock; + mutable std::mutex _sessionLock; SessionList _sessions; DurationSeconds _maxSessionRunTime; vespalib::string _baseDir; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index b7e02894e6b..830384ee538 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".transactionlog.domainpart"); @@ -14,7 +15,6 @@ using vespalib::FileHeader; using vespalib::string; using vespalib::getLastErrorString; using vespalib::IllegalHeaderException; -using vespalib::LockGuard; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::alloc::Alloc; @@ -238,7 +238,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) _range.to(packet.range().to()); _packets.insert(std::make_pair(firstSerial, std::move(packet))); { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _skipList.push_back(SkipInfo(firstSerial, firstPos)); } } else { @@ -329,7 +329,7 @@ DomainPart::close() { bool retval(false); { - LockGuard guard(_fileLock); + std::lock_guard guard(_fileLock); /* * Sync old domainpart before starting writing new, to avoid * hole. XXX: Feed latency spike due to lack of delayed open @@ -338,7 +338,7 @@ DomainPart::close() handleSync(*_transLog); _transLog->dropFromCache(); retval = _transLog->Close(); - LockGuard wguard(_writeLock); + std::lock_guard wguard(_writeLock); _syncedSerial = _writtenSerial; } if ( ! retval ) { @@ -346,7 +346,7 @@ DomainPart::close() _transLog->GetFileName(), _transLog->GetSize())); } { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _packets.clear(); } return retval; @@ -363,7 +363,7 @@ DomainPart::openAndFind(FastOS_FileInterface &file, const SerialNum &from) bool retval(file.OpenReadOnly(_transLog->GetFileName())); if (retval) { int64_t pos(_headerLen); - LockGuard guard(_lock); + std::lock_guard guard(_lock); for(SkipList::const_iterator it(_skipList.begin()), mt(_skipList.end()); (it < mt) && (it->id() <= from); it++) @@ -421,7 +421,7 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) } bool merged(false); - LockGuard guard(_lock); + std::lock_guard guard(_lock); if ( ! _packets.empty() ) { Packet & lastPacket = _packets.rbegin()->second; if (lastPacket.sizeBytes() < 0xf000) { @@ -440,12 +440,12 @@ DomainPart::sync() { SerialNum syncSerial(0); { - LockGuard guard(_writeLock); + std::lock_guard guard(_writeLock); syncSerial = _writtenSerial; } - LockGuard guard(_fileLock); + std::lock_guard guard(_fileLock); handleSync(*_transLog); - LockGuard wguard(_writeLock); + std::lock_guard wguard(_writeLock); if (_syncedSerial < syncSerial) { _syncedSerial = syncSerial; } @@ -455,7 +455,7 @@ bool DomainPart::visit(SerialNumRange &r, Packet &packet) { bool retval(false); - LockGuard guard(_lock); + std::lock_guard guard(_lock); LOG(spam, "Visit r(%" PRIu64 ", %" PRIu64 "] Checking %" PRIu64 " packets", r.from(), r.to(), uint64_t(_packets.size())); if ( ! isClosed() ) { @@ -549,7 +549,7 @@ DomainPart::write(FastOS_FileInterface &file, const IChunk & chunk) os << realEncoding.getRaw(); //Patching real encoding os << uint32_t(end - (begin + sizeof(uint32_t) + sizeof(uint8_t))); // Patching actual size. os.wp(end); - LockGuard guard(_writeLock); + std::lock_guard guard(_writeLock); if ( ! file.CheckedWrite(os.data(), os.size()) ) { throw runtime_error(handleWriteError("Failed writing the entry.", file, byteSize(), chunk.range(), os.size())); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h index 31d6938b654..a956932be19 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h @@ -3,11 +3,11 @@ #include "common.h" #include "ichunk.h" -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/memory.h> #include <map> #include <vector> #include <atomic> +#include <mutex> class FastOS_FileInterface; @@ -34,7 +34,7 @@ public: SerialNumRange range() const { return _range; } SerialNum getSynced() const { - vespalib::LockGuard guard(_writeLock); + std::lock_guard guard(_writeLock); return _syncedSerial; } @@ -74,8 +74,8 @@ private: typedef std::map<SerialNum, Packet> PacketList; const Encoding _encoding; const uint8_t _compressionLevel; - vespalib::Lock _lock; - vespalib::Lock _fileLock; + std::mutex _lock; + std::mutex _fileLock; SerialNumRange _range; size_t _sz; std::atomic<uint64_t> _byteSize; @@ -84,7 +84,7 @@ private: std::unique_ptr<FastOS_FileInterface> _transLog; SkipList _skipList; uint32_t _headerLen; - vespalib::Lock _writeLock; + mutable std::mutex _writeLock; // Protected by _writeLock SerialNum _writtenSerial; SerialNum _syncedSerial; diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.cpp b/searchlib/src/vespa/searchlib/transactionlog/session.cpp index c91b719be37..3aedeb11121 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/session.cpp @@ -3,8 +3,9 @@ #include "domain.h" #include "domainpart.h" #include <vespa/fastlib/io/bufferedfile.h> -#include <vespa/log/log.h> +#include <cassert> +#include <vespa/log/log.h> LOG_SETUP(".transactionlog.session"); using vespalib::LockGuard; diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.h b/searchlib/src/vespa/searchlib/transactionlog/session.h index ddbe218ed4e..ea78907da90 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.h +++ b/searchlib/src/vespa/searchlib/transactionlog/session.h @@ -3,7 +3,6 @@ #include "common.h" #include <vespa/vespalib/util/executor.h> -#include <vespa/vespalib/util/sync.h> #include <chrono> #include <deque> #include <atomic> diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h index 289a0fcb8c0..c232dfdad69 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h @@ -3,10 +3,10 @@ #include "client_common.h" #include "client_session.h" -#include <vespa/vespalib/util/sync.h> #include <vespa/fnet/frt/invokable.h> #include <map> #include <vector> +#include <mutex> class FNET_Transport; class FRT_Supervisor; @@ -58,7 +58,7 @@ private: vespalib::string _rpcTarget; SessionMap _sessions; //Brute force lock for subscriptions. For multithread safety. - vespalib::Lock _lock; + std::mutex _lock; std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _supervisor; diff --git a/slobrok/src/tests/configure/configure.cpp b/slobrok/src/tests/configure/configure.cpp index aa9826045ef..0e5b68053c8 100644 --- a/slobrok/src/tests/configure/configure.cpp +++ b/slobrok/src/tests/configure/configure.cpp @@ -5,11 +5,11 @@ #include <vespa/slobrok/sbregister.h> #include <vespa/slobrok/server/slobrokserver.h> #include <vespa/config/config.h> +#include <vespa/config/common/configcontext.h> #include <vespa/config-slobroks.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/vespalib/util/host_name.h> -#include <sstream> #include <algorithm> #include <iostream> @@ -125,7 +125,7 @@ TEST("configure_test") { set.addBuilder("client2", &cli2Builder); set.addBuilder("client3", &cli3Builder); - config::IConfigContext::SP cfgCtx(new config::ConfigContext(set)); + auto cfgCtx = std::make_shared<config::ConfigContext>(set); ConfigShim srvConfig1(18524, "server1", cfgCtx); ConfigShim srvConfig2(18525, "server2", cfgCtx); diff --git a/slobrok/src/tests/mirrorapi/mirrorapi.cpp b/slobrok/src/tests/mirrorapi/mirrorapi.cpp index 53e194fad2d..4126f34716f 100644 --- a/slobrok/src/tests/mirrorapi/mirrorapi.cpp +++ b/slobrok/src/tests/mirrorapi/mirrorapi.cpp @@ -140,7 +140,7 @@ Test::Main() FastOS_ThreadPool threadPool(0x10000); FNET_Transport transport; FRT_Supervisor supervisor(&transport); - MirrorAPI mirror(supervisor, config::ConfigUri::createFromInstance(specBuilder)); + MirrorAPI mirror(supervisor, slobrok::ConfiguratorFactory(config::ConfigUri::createFromInstance(specBuilder))); EXPECT_TRUE(!mirror.ready()); transport.Start(&threadPool); std::this_thread::sleep_for(1s); diff --git a/slobrok/src/vespa/slobrok/cfg.h b/slobrok/src/vespa/slobrok/cfg.h index 173f08c3227..db83790677a 100644 --- a/slobrok/src/vespa/slobrok/cfg.h +++ b/slobrok/src/vespa/slobrok/cfg.h @@ -31,9 +31,9 @@ class ConfiguratorFactory { private: config::ConfigUri _uri; public: - ConfiguratorFactory(const config::ConfigUri & uri); + explicit ConfiguratorFactory(const config::ConfigUri & uri); // Convenience. Might belong somewhere else - ConfiguratorFactory(const std::vector<std::string> & spec); + explicit ConfiguratorFactory(const std::vector<std::string> & spec); Configurator::UP create(Configurable &target) const; }; diff --git a/slobrok/src/vespa/slobrok/sblist.cpp b/slobrok/src/vespa/slobrok/sblist.cpp index 9fc58aa2974..e7528fb6db2 100644 --- a/slobrok/src/vespa/slobrok/sblist.cpp +++ b/slobrok/src/vespa/slobrok/sblist.cpp @@ -5,7 +5,7 @@ #include <vespa/log/log.h> LOG_SETUP(".slobrok.list"); -using vespalib::LockGuard; +using LockGuard = std::lock_guard<std::mutex>; namespace slobrok::api { @@ -83,7 +83,7 @@ SlobrokList::setup(const std::vector<std::string> &specList) _slobrokSpecs.push_back(specList[i]); } - vespalib::RandomGen randomizer(time(NULL)); + vespalib::RandomGen randomizer(time(nullptr)); // randomize order for (size_t i = 0; i + 1 < cfgSz; ++i) { size_t lim = cfgSz - i; diff --git a/slobrok/src/vespa/slobrok/sblist.h b/slobrok/src/vespa/slobrok/sblist.h index e5f31c23e5b..08328b99d1a 100644 --- a/slobrok/src/vespa/slobrok/sblist.h +++ b/slobrok/src/vespa/slobrok/sblist.h @@ -2,7 +2,6 @@ #pragma once #include "cfg.h" -#include <vespa/vespalib/util/sync.h> namespace slobrok::api { @@ -42,7 +41,7 @@ public: /** check if the list contains a given spec */ bool contains(const std::string &spec); private: - vespalib::Lock _lock; + std::mutex _lock; std::vector<std::string> _slobrokSpecs; size_t _nextSpec; size_t _currSpec; diff --git a/slobrok/src/vespa/slobrok/sbmirror.cpp b/slobrok/src/vespa/slobrok/sbmirror.cpp index f0a810dad10..855ed4dd82d 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.cpp +++ b/slobrok/src/vespa/slobrok/sbmirror.cpp @@ -55,7 +55,7 @@ MirrorAPI::SpecList MirrorAPI::lookup(const std::string & pattern) const { SpecList ret; - LockGuard guard(_lock); + std::lock_guard guard(_lock); SpecList::const_iterator end = _specs.end(); for (SpecList::const_iterator it = _specs.begin(); it != end; ++it) { if (match(it->first.c_str(), pattern.c_str())) { @@ -104,7 +104,7 @@ void MirrorAPI::updateTo(SpecList& newSpecs, uint32_t newGen) { { - LockGuard guard(_lock); + std::lock_guard guard(_lock); std::swap(newSpecs, _specs); _updates.add(); } @@ -117,7 +117,7 @@ MirrorAPI::updateTo(SpecList& newSpecs, uint32_t newGen) bool MirrorAPI::ready() const { - LockGuard guard(_lock); + std::lock_guard guard(_lock); return _updates.getAsInt() != 0; } diff --git a/slobrok/src/vespa/slobrok/sbmirror.h b/slobrok/src/vespa/slobrok/sbmirror.h index 152886fc02e..437ef334af6 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.h +++ b/slobrok/src/vespa/slobrok/sbmirror.h @@ -88,7 +88,7 @@ private: void reSched(double seconds); FRT_Supervisor &_orb; - mutable vespalib::Lock _lock; + mutable std::mutex _lock; bool _reqPending; bool _scheduled; bool _reqDone; diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt index f2f8a41b68d..efb57618823 100644 --- a/staging_vespalib/CMakeLists.txt +++ b/staging_vespalib/CMakeLists.txt @@ -23,7 +23,6 @@ vespa_define_module( src/tests/floatingpointtype src/tests/growablebytebuffer src/tests/json - src/tests/librarypool src/tests/memorydatastore src/tests/metrics src/tests/objectdump diff --git a/staging_vespalib/src/tests/librarypool/.gitignore b/staging_vespalib/src/tests/librarypool/.gitignore deleted file mode 100644 index 1a1aea2fda0..00000000000 --- a/staging_vespalib/src/tests/librarypool/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_librarypool_test_app diff --git a/staging_vespalib/src/tests/librarypool/CMakeLists.txt b/staging_vespalib/src/tests/librarypool/CMakeLists.txt deleted file mode 100644 index 83e1e92e680..00000000000 --- a/staging_vespalib/src/tests/librarypool/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_librarypool_test_app TEST - SOURCES - librarypool_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_librarypool_test_app COMMAND staging_vespalib_librarypool_test_app) diff --git a/staging_vespalib/src/tests/librarypool/librarypool_test.cpp b/staging_vespalib/src/tests/librarypool/librarypool_test.cpp deleted file mode 100644 index adefdf3aa6b..00000000000 --- a/staging_vespalib/src/tests/librarypool/librarypool_test.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/librarypool.h> -#include <vespa/vespalib/util/exceptions.h> - -using namespace vespalib; - -class Test : public TestApp -{ -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("librarypool_test"); - LibraryPool p; - ASSERT_TRUE(p.get("z") == NULL); - p.loadLibrary("z"); - ASSERT_TRUE(p.get("z") != NULL); - ASSERT_TRUE(p.get("z")->GetSymbol("some_symbol_that_is_not_there") == NULL); - ASSERT_TRUE(p.get("z")->GetSymbol("compress") != NULL); - try { - p.loadLibrary("not_found"); - ASSERT_TRUE(false); - } catch (const IllegalArgumentException & e) { - ASSERT_TRUE(p.get("not_found") == NULL); - } - { - const LibraryPool & c(p); - ASSERT_TRUE(c.get("z") != NULL); - ASSERT_TRUE(c.get("not_found") == NULL); - } - TEST_DONE(); -} - -TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/metrics/mock_tick.h b/staging_vespalib/src/tests/metrics/mock_tick.h index 3f244ea6c9f..d2e985b5ba3 100644 --- a/staging_vespalib/src/tests/metrics/mock_tick.h +++ b/staging_vespalib/src/tests/metrics/mock_tick.h @@ -15,8 +15,9 @@ class MockTick : public Tick { private: using Guard = std::unique_lock<std::mutex>; struct Value { - TimeStamp value{0.0}; - bool valid{false}; + Value() noexcept : value(0.0), valid(false) {} + TimeStamp value; + bool valid; }; TimeStamp _first_value; @@ -55,7 +56,7 @@ private: } public: - MockTick(TimeStamp first_value) + explicit MockTick(TimeStamp first_value) noexcept : _first_value(first_value), _lock(), _cond(), _alive(true), _prev(), _next() {} TimeStamp first() override { return _first_value; } TimeStamp next(TimeStamp prev) override { @@ -81,7 +82,7 @@ class TickProxy : public Tick { private: std::shared_ptr<Tick> _tick; public: - TickProxy(std::shared_ptr<Tick> tick) : _tick(std::move(tick)) {} + explicit TickProxy(std::shared_ptr<Tick> tick) noexcept : _tick(std::move(tick)) {} TimeStamp first() override { return _tick->first(); } TimeStamp next(TimeStamp prev) override { return _tick->next(prev); } bool alive() const override { return _tick->alive(); } diff --git a/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp b/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp index e3ff7533d58..49bc52d6239 100644 --- a/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp +++ b/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp @@ -7,7 +7,7 @@ using namespace vespalib; class A { public: - virtual ~A() { } + virtual ~A() = default; virtual void assign(const A & rhs) { (void) rhs; assert(false); } // Required by the primitive array. virtual A * clone() const { assert(false); return nullptr; } // Required for the complex array. @@ -19,7 +19,7 @@ public: class Primitive : public A { public: - Primitive(size_t v=11) : _v(v) { } + Primitive(size_t v=11) noexcept : _v(v) { } size_t value() const { return _v; } bool operator == (const A & rhs) const override { return dynamic_cast<const Primitive &>(rhs).value() == value(); @@ -38,7 +38,7 @@ private: class Complex : public A { public: - Complex(size_t v=11) : _v(v) { } + Complex(size_t v=11) noexcept : _v(v) { } size_t value() const { return _v; } bool operator == (const A & rhs) const override { return dynamic_cast<const Complex &>(rhs).value() == value(); diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp index 2ca49105610..e75cdd09c8c 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp @@ -31,7 +31,7 @@ public: int _fail; int _val; - TestObj() + TestObj() noexcept : _m(), _cv(), _done(0), diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp index a2671bb81a7..03ec64b771e 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp @@ -33,7 +33,7 @@ public: int _fail; int _val; - TestObj() + TestObj() noexcept : _m(), _cv(), _done(0), diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index df94e70f9d6..29b25cd0471 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -33,7 +33,7 @@ public: int _fail; int _val; - TestObj() + TestObj() noexcept : _m(), _cv(), _done(0), diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp index cd7a3abb1eb..7909ec7486e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp +++ b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp @@ -1,10 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dummy_metrics_manager.h" -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { -DummyMetricsManager::~DummyMetricsManager() {} +DummyMetricsManager::~DummyMetricsManager() = default; Snapshot DummyMetricsManager::snapshot() @@ -20,5 +19,4 @@ DummyMetricsManager::totalSnapshot() return snap; } -} // namespace vespalib::metrics -} // namespace vespalib +} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h index 6aeb1137732..1536888014a 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h @@ -10,8 +10,7 @@ #include "metrics_manager.h" #include "clock.h" -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { /** * Dummy manager that discards everything, use @@ -21,9 +20,9 @@ namespace metrics { class DummyMetricsManager : public MetricsManager { protected: - DummyMetricsManager() {} + DummyMetricsManager() noexcept {} public: - ~DummyMetricsManager(); + ~DummyMetricsManager() override; static std::shared_ptr<MetricsManager> create() { return std::shared_ptr<MetricsManager>(new DummyMetricsManager()); @@ -58,5 +57,4 @@ public: void sample(Gauge::Measurement) override {} }; -} // namespace vespalib::metrics -} // namespace vespalib +} diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp index d016b052dae..d4fe0ef43e1 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp +++ b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp @@ -10,27 +10,28 @@ SimpleComponentConfigProducer::SimpleComponentConfigProducer() { } +SimpleComponentConfigProducer::~SimpleComponentConfigProducer() = default; + void SimpleComponentConfigProducer::addConfig(const Config &config) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _state.insert(std::make_pair(config.name, config)).first->second = config; } void SimpleComponentConfigProducer::removeConfig(const vespalib::string &name) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _state.erase(name); } void SimpleComponentConfigProducer::getComponentConfig(Consumer &consumer) { - typedef std::map<vespalib::string, Config>::const_iterator ITR; - LockGuard guard(_lock); - for (ITR itr = _state.begin(); itr != _state.end(); ++itr) { - consumer.add(itr->second); + std::lock_guard guard(_lock); + for (const auto & entry : _state) { + consumer.add(entry.second); } } diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h index 5783139c9d5..70dd00a8792 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h +++ b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h @@ -3,19 +3,20 @@ #pragma once #include "component_config_producer.h" -#include <vespa/vespalib/util/sync.h> #include <map> +#include <mutex> namespace vespalib { class SimpleComponentConfigProducer : public ComponentConfigProducer { private: - Lock _lock; + std::mutex _lock; std::map<vespalib::string, Config> _state; public: SimpleComponentConfigProducer(); + ~SimpleComponentConfigProducer() override; void addConfig(const Config &config); void removeConfig(const vespalib::string &name); void getComponentConfig(Consumer &consumer) override; diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp index 651dab97e68..a25888399c1 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp +++ b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp @@ -4,7 +4,6 @@ #include <vespa/defaults.h> #include <fcntl.h> #include <sys/stat.h> -#include <sys/types.h> #include <unistd.h> namespace { @@ -51,28 +50,26 @@ SimpleHealthProducer::SimpleHealthProducer() setOk(); } -SimpleHealthProducer::~SimpleHealthProducer() -{ -} +SimpleHealthProducer::~SimpleHealthProducer() = default; void SimpleHealthProducer::setOk() { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _health = Health(true, "All OK"); } void SimpleHealthProducer::setFailed(const vespalib::string &msg) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _health = Health(false, msg); } HealthProducer::Health SimpleHealthProducer::getHealth() const { - LockGuard guard(_lock); + std::lock_guard guard(_lock); if (_health.ok && diskFailed()) { return Health(false, "disk ping failed"); } diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h index 90fe2489b0a..fc89f5c7644 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h +++ b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h @@ -3,19 +3,19 @@ #pragma once #include "health_producer.h" -#include <vespa/vespalib/util/sync.h> +#include <mutex> namespace vespalib { class SimpleHealthProducer : public HealthProducer { private: - Lock _lock; + mutable std::mutex _lock; HealthProducer::Health _health; public: SimpleHealthProducer(); - ~SimpleHealthProducer(); + ~SimpleHealthProducer() override; void setOk(); void setFailed(const vespalib::string &msg); Health getHealth() const override; diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp index 97be2a61235..b39bc96a5b6 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp +++ b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp @@ -11,35 +11,33 @@ SimpleMetricsProducer::SimpleMetricsProducer() { } -SimpleMetricsProducer::~SimpleMetricsProducer() -{ -} +SimpleMetricsProducer::~SimpleMetricsProducer() = default; void SimpleMetricsProducer::setMetrics(const vespalib::string &metrics) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _metrics = metrics; } vespalib::string SimpleMetricsProducer::getMetrics(const vespalib::string &) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); return _metrics; } void SimpleMetricsProducer::setTotalMetrics(const vespalib::string &metrics) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _totalMetrics = metrics; } vespalib::string SimpleMetricsProducer::getTotalMetrics(const vespalib::string &) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); return _totalMetrics; } diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h index 1dd1452c32f..fdcf1dce6b6 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h +++ b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h @@ -3,24 +3,24 @@ #pragma once #include "metrics_producer.h" -#include <vespa/vespalib/util/sync.h> +#include <mutex> namespace vespalib { class SimpleMetricsProducer : public MetricsProducer { private: - Lock _lock; + std::mutex _lock; vespalib::string _metrics; vespalib::string _totalMetrics; public: SimpleMetricsProducer(); - ~SimpleMetricsProducer(); + ~SimpleMetricsProducer() override; void setMetrics(const vespalib::string &metrics); - virtual vespalib::string getMetrics(const vespalib::string &consumer) override; + vespalib::string getMetrics(const vespalib::string &consumer) override; void setTotalMetrics(const vespalib::string &metrics); - virtual vespalib::string getTotalMetrics(const vespalib::string &consumer) override; + vespalib::string getTotalMetrics(const vespalib::string &consumer) override; }; } // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.hpp b/staging_vespalib/src/vespa/vespalib/objects/identifiable.hpp index 2c34ba306ab..b1c4a128feb 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.hpp +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.hpp @@ -46,16 +46,16 @@ class IdentifiablePtr : public CloneablePtr<T> { public: IdentifiablePtr(const T &t) : CloneablePtr<T>(t.clone()) {} - IdentifiablePtr(IdentifiablePtr &&) = default; - IdentifiablePtr & operator = (IdentifiablePtr &&) = default; + IdentifiablePtr(IdentifiablePtr &&) noexcept = default; + IdentifiablePtr & operator = (IdentifiablePtr &&) noexcept = default; IdentifiablePtr(const IdentifiablePtr &) = default; IdentifiablePtr & operator = (const IdentifiablePtr &) = default; - IdentifiablePtr(T * p=NULL) : CloneablePtr<T>(p) { } - IdentifiablePtr(std::unique_ptr<T> &&rhs) + IdentifiablePtr(T * p=NULL) noexcept : CloneablePtr<T>(p) { } + IdentifiablePtr(std::unique_ptr<T> &&rhs) noexcept : CloneablePtr<T>(std::move(rhs)) { } - IdentifiablePtr &operator=(std::unique_ptr<T> &&rhs) + IdentifiablePtr &operator=(std::unique_ptr<T> &&rhs) noexcept { CloneablePtr<T>::operator=(std::move(rhs)); return *this; diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.h b/staging_vespalib/src/vespa/vespalib/stllike/cache.h index 99b601f7c3c..798fdb70138 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/cache.h +++ b/staging_vespalib/src/vespa/vespalib/stllike/cache.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/stllike/lrucache_map.h> -#include <vespa/vespalib/util/sync.h> #include <atomic> namespace vespalib { @@ -42,7 +41,7 @@ struct CacheParam : public P template< typename P > class cache : private lrucache_map<P> { - typedef lrucache_map<P> Lru; + using Lru = lrucache_map<P>; protected: typedef typename P::BackingStore BackingStore; typedef typename P::Hash Hash; @@ -120,11 +119,12 @@ public: size_t getlookup() const { return _lookup; } protected: - vespalib::LockGuard getGuard(); - void invalidate(const vespalib::LockGuard & guard, const K & key); - bool hasKey(const vespalib::LockGuard & guard, const K & key) const; - bool hasLock() const; + using UniqueLock = std::unique_lock<std::mutex>; + UniqueLock getGuard(); + void invalidate(const UniqueLock & guard, const K & key); + bool hasKey(const UniqueLock & guard, const K & key) const; private: + void verifyHashLock(const UniqueLock & guard) const; /** * Called when an object is inserted, to see if the LRU should be removed. * Default is to obey the maxsize given in constructor. @@ -133,7 +133,7 @@ private: */ bool removeOldest(const value_type & v) override; size_t calcSize(const K & k, const V & v) const { return sizeof(value_type) + _sizeK(k) + _sizeV(v); } - vespalib::Lock & getLock(const K & k) { + std::mutex & getLock(const K & k) { size_t h(_hasher(k)); return _addLocks[h%(sizeof(_addLocks)/sizeof(_addLocks[0]))]; } @@ -153,9 +153,9 @@ private: mutable size_t _invalidate; mutable size_t _lookup; BackingStore & _store; - vespalib::Lock _hashLock; + mutable std::mutex _hashLock; /// Striped locks that can be used for having a locked access to the backing store. - vespalib::Lock _addLocks[113]; + std::mutex _addLocks[113]; }; } diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp b/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp index 906621d623c..719169ba747 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp +++ b/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp @@ -30,25 +30,19 @@ cache<P>::setCapacityBytes(size_t sz) { template< typename P > void cache<P>::invalidate(const K & key) { - vespalib::LockGuard guard(_hashLock); + UniqueLock guard(_hashLock); invalidate(guard, key); } template< typename P > bool cache<P>::hasKey(const K & key) const { - vespalib::LockGuard guard(_hashLock); + UniqueLock guard(_hashLock); return hasKey(guard, key); } template< typename P > -bool -cache<P>::hasLock() const { - return TryLock(_hashLock).hasLock(); -} - -template< typename P > -cache<P>::~cache() { } +cache<P>::~cache() = default; template< typename P > cache<P>::cache(BackingStore & b, size_t maxBytes) : @@ -79,9 +73,9 @@ cache<P>::removeOldest(const value_type & v) { } template< typename P > -vespalib::LockGuard +std::unique_lock<std::mutex> cache<P>::getGuard() { - return vespalib::LockGuard(_hashLock); + return UniqueLock(_hashLock); } template< typename P > @@ -89,7 +83,7 @@ typename P::Value cache<P>::read(const K & key) { { - vespalib::LockGuard guard(_hashLock); + std::lock_guard guard(_hashLock); if (Lru::hasKey(key)) { _hit++; return (*this)[key]; @@ -98,9 +92,9 @@ cache<P>::read(const K & key) } } - vespalib::LockGuard storeGuard(getLock(key)); + std::lock_guard storeGuard(getLock(key)); { - vespalib::LockGuard guard(_hashLock); + std::lock_guard guard(_hashLock); if (Lru::hasKey(key)) { // Somebody else just fetched it ahead of me. _race++; @@ -109,7 +103,7 @@ cache<P>::read(const K & key) } V value; if (_store.read(key, value)) { - vespalib::LockGuard guard(_hashLock); + std::lock_guard guard(_hashLock); Lru::insert(key, value); _sizeBytes += calcSize(key, value); _insert++; @@ -124,9 +118,9 @@ void cache<P>::write(const K & key, V value) { size_t newSize = calcSize(key, value); - vespalib::LockGuard storeGuard(getLock(key)); + std::lock_guard storeGuard(getLock(key)); { - vespalib::LockGuard guard(_hashLock); + std::lock_guard guard(_hashLock); if (Lru::hasKey(key)) { _sizeBytes -= calcSize(key, (*this)[key]); _update++; @@ -135,7 +129,7 @@ cache<P>::write(const K & key, V value) _store.write(key, value); { - vespalib::LockGuard guard(_hashLock); + std::lock_guard guard(_hashLock); (*this)[key] = std::move(value); _sizeBytes += newSize; _write++; @@ -146,17 +140,16 @@ template< typename P > void cache<P>::erase(const K & key) { - vespalib::LockGuard storeGuard(getLock(key)); + std::lock_guard storeGuard(getLock(key)); invalidate(key); _store.erase(key); } template< typename P > void -cache<P>::invalidate(const vespalib::LockGuard & guard, const K & key) +cache<P>::invalidate(const UniqueLock & guard, const K & key) { - assert(guard.locks(_hashLock)); - (void) guard; + verifyHashLock(guard); if (Lru::hasKey(key)) { _sizeBytes -= calcSize(key, (*this)[key]); _invalidate++; @@ -166,13 +159,19 @@ cache<P>::invalidate(const vespalib::LockGuard & guard, const K & key) template< typename P > bool -cache<P>::hasKey(const vespalib::LockGuard & guard, const K & key) const +cache<P>::hasKey(const UniqueLock & guard, const K & key) const { - (void) guard; - assert(guard.locks(_hashLock)); + verifyHashLock(guard); _lookup++; return Lru::hasKey(key); } +template< typename P > +void +cache<P>::verifyHashLock(const UniqueLock & guard) const { + assert(guard.mutex() == & _hashLock); + assert(guard.owns_lock()); +} + } diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt index 586e06396e7..70f17f76e4c 100644 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT jsonexception.cpp jsonstream.cpp jsonwriter.cpp - librarypool.cpp process_memory_stats.cpp programoptions.cpp programoptions_testutils.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp index c0a32197c07..0f7e6155276 100644 --- a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp @@ -2,6 +2,7 @@ #include "document_runnable.h" #include <vespa/vespalib/util/exceptions.h> +#include <cassert> namespace document { diff --git a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp deleted file mode 100644 index 2a3ca21c369..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/util/librarypool.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/stringfmt.h> - -namespace vespalib { - -LibraryPool::LibraryPool() : - _libraries(), - _lock() -{ -} - -LibraryPool::~LibraryPool() -{ - LockGuard guard(_lock); - _libraries.clear(); -} - -void -LibraryPool::loadLibrary(stringref libName) -{ - LockGuard guard(_lock); - if (_libraries.find(libName) == _libraries.end()) { - DynamicLibrarySP lib(new FastOS_DynamicLibrary); - string file(libName); - if (!lib->Open(file.c_str())) { - string error = lib->GetLastErrorString(); - throw IllegalArgumentException(make_string("Failed loading dynamic library '%s' due to '%s'.", - file.c_str(), error.c_str())); - } else { - _libraries[libName] = lib; - } - } -} - -FastOS_DynamicLibrary * -LibraryPool::get(stringref name) -{ - LockGuard guard(_lock); - LibraryMap::const_iterator found(_libraries.find(name)); - return (found != _libraries.end()) - ? found->second.get() - : NULL; -} - -const FastOS_DynamicLibrary * -LibraryPool::get(stringref name) const -{ - LockGuard guard(_lock); - LibraryMap::const_iterator found(_libraries.find(name)); - return (found != _libraries.end()) - ? found->second.get() - : NULL; -} - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/librarypool.h b/staging_vespalib/src/vespa/vespalib/util/librarypool.h deleted file mode 100644 index f9149589338..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/librarypool.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/fastos/dynamiclibrary.h> -#include <map> - -namespace vespalib { - -class LibraryPool -{ -public: - LibraryPool(); - ~LibraryPool(); - /** - * This will load the library with the name given. - * - It will verify linkage at load time. - * - Symbols will be private. - * @param name The name of the library to load. That is without the 'lib' prefix and the '.so' extension. - * @throws IllegalArgumentException if there are any errors. - */ - void loadLibrary(stringref name); - /** - * Will return the library requested. NULL if not found. - * @param name The name of the library as given in the @ref loadLibrary call. - * @return The library that has already been loaded. NULL if not found. - */ - FastOS_DynamicLibrary *get(stringref name); - const FastOS_DynamicLibrary *get(stringref name) const; -private: - typedef std::shared_ptr<FastOS_DynamicLibrary> DynamicLibrarySP; - typedef std::map<vespalib::string, DynamicLibrarySP> LibraryMap; - LibraryMap _libraries; - Lock _lock; -}; - -} - diff --git a/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.cpp index d9b4feda293..3f5b3b79656 100644 --- a/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.cpp @@ -45,7 +45,7 @@ ScheduledExecutor::ScheduledExecutor() ScheduledExecutor::~ScheduledExecutor() { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _transport->ShutDown(true); _threadPool.Close(); _taskList.clear(); @@ -55,7 +55,7 @@ ScheduledExecutor::~ScheduledExecutor() void ScheduledExecutor::scheduleAtFixedRate(vespalib::Executor::Task::UP task, duration delay, duration interval) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); TimerTaskPtr tTask(new TimerTask(_transport->GetScheduler(), std::move(task), interval)); _taskList.push_back(std::move(tTask)); _taskList.back()->Schedule(to_s(delay)); @@ -64,10 +64,10 @@ ScheduledExecutor::scheduleAtFixedRate(vespalib::Executor::Task::UP task, durati void ScheduledExecutor::reset() { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _transport->ShutDown(true); _taskList.clear(); - _transport.reset(new FNET_Transport()); + _transport = std::make_unique<FNET_Transport>(); _transport->Start(&_threadPool); } diff --git a/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.h b/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.h index 0f052c762a7..0b98a236f74 100644 --- a/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/util/executor.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/time.h> #include <vespa/fastos/thread.h> #include <vector> @@ -25,8 +24,8 @@ private: typedef std::vector<TimerTaskPtr> TaskList; FastOS_ThreadPool _threadPool; std::unique_ptr<FNET_Transport> _transport; - vespalib::Lock _lock; - TaskList _taskList; + std::mutex _lock; + TaskList _taskList; public: /** diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp index 65857183879..795a7ef1ec3 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp @@ -1,7 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "singleexecutor.h" -#include <vespa/vespalib/util/time.h> +#include <cassert> namespace vespalib { diff --git a/staging_vespalib/src/vespa/vespalib/util/varholder.h b/staging_vespalib/src/vespa/vespalib/util/varholder.h index fdcc15d1fb4..5b68ffa5582 100644 --- a/staging_vespalib/src/vespa/vespalib/util/varholder.h +++ b/staging_vespalib/src/vespa/vespalib/util/varholder.h @@ -2,15 +2,15 @@ #pragma once -#include <vespa/vespalib/util/sync.h> +#include <mutex> namespace vespalib { template <typename T> class VarHolder { - T _v; - Lock _lock; + T _v; + mutable std::mutex _lock; public: VarHolder() : _v(), _lock() {} explicit VarHolder(const T &v) : _v(v), _lock() {} @@ -21,7 +21,7 @@ public: void set(const T &v) { T old; { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); old = _v; _v = v; } @@ -30,7 +30,7 @@ public: void clear() { set(T()); } T get() const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); return _v; } }; diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index 709e158e531..527c0a927b4 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -148,7 +148,7 @@ namespace { template <typename Map> struct NonConstProcessor { - typename Map::Decision operator()(int key, A& a) { + typename Map::Decision operator() (int key, A& a) const noexcept { (void) key; ++a._val2; return Map::UPDATE; diff --git a/storage/src/tests/common/dummystoragelink.cpp b/storage/src/tests/common/dummystoragelink.cpp index a747fbcf998..ab70bff3409 100644 --- a/storage/src/tests/common/dummystoragelink.cpp +++ b/storage/src/tests/common/dummystoragelink.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dummystoragelink.h" #include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <sys/time.h> #include <vespa/vespalib/util/exceptions.h> +#include <cassert> namespace storage { -DummyStorageLink* DummyStorageLink::_last(0); +DummyStorageLink* DummyStorageLink::_last(nullptr); DummyStorageLink::DummyStorageLink() : StorageLink("Dummy storage link"), @@ -37,7 +37,7 @@ DummyStorageLink::~DummyStorageLink() bool DummyStorageLink::handleInjectedReply() { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (!_injected.empty()) { sendUp(*_injected.begin()); _injected.pop_front(); @@ -65,7 +65,7 @@ bool DummyStorageLink::onDown(const api::StorageMessage::SP& cmd) if (isBottom()) { vespalib::MonitorGuard lock(_waitMonitor); { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _commands.push_back(cmd); } lock.broadcast(); @@ -78,7 +78,7 @@ bool DummyStorageLink::onUp(const api::StorageMessage::SP& reply) { if (isTop()) { vespalib::MonitorGuard lock(_waitMonitor); { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _replies.push_back(reply); } lock.broadcast(); @@ -91,13 +91,13 @@ bool DummyStorageLink::onUp(const api::StorageMessage::SP& reply) { void DummyStorageLink::injectReply(api::StorageReply* reply) { assert(reply); - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _injected.push_back(std::shared_ptr<api::StorageReply>(reply)); } void DummyStorageLink::reset() { vespalib::MonitorGuard lock(_waitMonitor); - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _commands.clear(); _replies.clear(); _injected.clear(); diff --git a/storage/src/tests/common/dummystoragelink.h b/storage/src/tests/common/dummystoragelink.h index 174f8219560..46a1d4ea25f 100644 --- a/storage/src/tests/common/dummystoragelink.h +++ b/storage/src/tests/common/dummystoragelink.h @@ -18,7 +18,7 @@ namespace storage { class DummyStorageLink : public StorageLink { - mutable vespalib::Lock _lock; // to protect below containers: + mutable std::mutex _lock; // to protect below containers: std::vector<api::StorageMessage::SP> _commands; std::vector<api::StorageMessage::SP> _replies; std::list<api::StorageMessage::SP> _injected; @@ -31,7 +31,7 @@ class DummyStorageLink : public StorageLink { public: DummyStorageLink(); - ~DummyStorageLink(); + ~DummyStorageLink() override; bool onDown(const api::StorageMessage::SP&) override; bool onUp(const api::StorageMessage::SP&) override; @@ -63,21 +63,21 @@ public: void waitForMessage(const api::MessageType&, int timeout = -1); api::StorageMessage::SP getCommand(size_t i) const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); api::StorageMessage::SP ret = _commands[i]; return ret; } api::StorageMessage::SP getReply(size_t i) const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); api::StorageMessage::SP ret = _replies[i]; return ret; } size_t getNumCommands() const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); return _commands.size(); } size_t getNumReplies() const { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); return _replies.size(); } @@ -90,7 +90,7 @@ public: vespalib::MonitorGuard lock(_waitMonitor); std::vector<api::StorageMessage::SP> retval; { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); retval.swap(_commands); } return retval; @@ -100,7 +100,7 @@ public: vespalib::MonitorGuard lock(_waitMonitor); std::vector<api::StorageMessage::SP> retval; { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); retval.swap(_replies); } return retval; diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index 1847de0e84f..f2989f88765 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -57,9 +57,9 @@ TestStorageApp::TestStorageApp(StorageComponentRegisterImpl::UP compReg, _compReg.setPriorityConfig( *config::ConfigGetter<StorageComponent::PriorityConfig> ::getConfig(uri.getConfigId(), uri.getContext())); - loadTypes.reset(new documentapi::LoadTypeSet( + loadTypes = std::make_shared<documentapi::LoadTypeSet>( *config::ConfigGetter<vespa::config::content::LoadTypeConfig> - ::getConfig(uri.getConfigId(), uri.getContext()))); + ::getConfig(uri.getConfigId(), uri.getContext())); } else { if (index == 0xffff) index = 0; loadTypes.reset(new documentapi::LoadTypeSet); @@ -72,20 +72,18 @@ TestStorageApp::TestStorageApp(StorageComponentRegisterImpl::UP compReg, _compReg.setDocumentTypeRepo(_docMan.getTypeRepoSP()); _compReg.setLoadTypes(loadTypes); _compReg.setBucketIdFactory(document::BucketIdFactory()); - lib::Distribution::SP distr(new lib::Distribution( - lib::Distribution::getDefaultDistributionConfig( - redundancy, nodeCount))); + auto distr = std::make_shared<lib::Distribution>( + lib::Distribution::getDefaultDistributionConfig(redundancy, nodeCount)); _compReg.setDistribution(distr); } -TestStorageApp::~TestStorageApp() {} +TestStorageApp::~TestStorageApp() = default; void TestStorageApp::setDistribution(Redundancy redundancy, NodeCount nodeCount) { - lib::Distribution::SP distr(new lib::Distribution( - lib::Distribution::getDefaultDistributionConfig( - redundancy, nodeCount))); + auto distr = std::make_shared<lib::Distribution>( + lib::Distribution::getDefaultDistributionConfig(redundancy, nodeCount)); _compReg.setDistribution(distr); } @@ -98,8 +96,7 @@ TestStorageApp::setTypeRepo(std::shared_ptr<const document::DocumentTypeRepo> re void TestStorageApp::setClusterState(const lib::ClusterState& c) { - _nodeStateUpdater.setClusterState( - lib::ClusterState::CSP(new lib::ClusterState(c))); + _nodeStateUpdater.setClusterState(std::make_shared<lib::ClusterState>(c)); } void @@ -109,8 +106,7 @@ TestStorageApp::waitUntilInitialized( // Always use real clock for wait timeouts. Component clock may be faked // in tests framework::defaultimplementation::RealClock clock; - framework::MilliSecTime endTime( - clock.getTimeInMillis() + timeout.getMillis()); + framework::MilliSecTime endTime(clock.getTimeInMillis() + timeout.getMillis()); while (!isInitialized()) { std::this_thread::sleep_for(1ms); framework::MilliSecTime currentTime(clock.getTimeInMillis()); @@ -142,8 +138,7 @@ namespace { TestServiceLayerApp::TestServiceLayerApp(vespalib::stringref configId) : TestStorageApp(std::make_unique<ServiceLayerComponentRegisterImpl>(true), // TODO remove B-tree flag once default lib::NodeType::STORAGE, getIndexFromConfig(configId), configId), - _compReg(dynamic_cast<ServiceLayerComponentRegisterImpl&>( - TestStorageApp::getComponentRegister())), + _compReg(dynamic_cast<ServiceLayerComponentRegisterImpl&>(TestStorageApp::getComponentRegister())), _persistenceProvider(), _partitions(1) { @@ -157,8 +152,7 @@ TestServiceLayerApp::TestServiceLayerApp(DiskCount dc, NodeIndex index, vespalib::stringref configId) : TestStorageApp(std::make_unique<ServiceLayerComponentRegisterImpl>(true), // TODO remove B-tree flag once default lib::NodeType::STORAGE, index, configId), - _compReg(dynamic_cast<ServiceLayerComponentRegisterImpl&>( - TestStorageApp::getComponentRegister())), + _compReg(dynamic_cast<ServiceLayerComponentRegisterImpl&>(TestStorageApp::getComponentRegister())), _persistenceProvider(), _partitions(dc) { @@ -192,9 +186,7 @@ spi::PersistenceProvider& TestServiceLayerApp::getPersistenceProvider() { if (_persistenceProvider.get() == 0) { - throw vespalib::IllegalStateException( - "Persistence provider requested but not initialized.", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Persistence provider requested but not initialized.", VESPA_STRLOC); } return *_persistenceProvider; } @@ -203,9 +195,7 @@ spi::PartitionStateList& TestServiceLayerApp::getPartitions() { if (_persistenceProvider.get() == 0) { - throw vespalib::IllegalStateException( - "Partition list requested but not initialized.", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Partition list requested but not initialized.", VESPA_STRLOC); } return _partitions; } @@ -224,8 +214,7 @@ namespace { template<typename T> const T getConfig(vespalib::stringref configId) { config::ConfigUri uri(configId); - return *config::ConfigGetter<T>::getConfig( - uri.getConfigId(), uri.getContext()); + return *config::ConfigGetter<T>::getConfig(uri.getConfigId(), uri.getContext()); } } @@ -243,8 +232,7 @@ TestDistributorApp::TestDistributorApp(vespalib::stringref configId) : TestStorageApp( std::make_unique<DistributorComponentRegisterImpl>(), lib::NodeType::DISTRIBUTOR, getIndexFromConfig(configId), configId), - _compReg(dynamic_cast<DistributorComponentRegisterImpl&>( - TestStorageApp::getComponentRegister())), + _compReg(dynamic_cast<DistributorComponentRegisterImpl&>(TestStorageApp::getComponentRegister())), _lastUniqueTimestampRequested(0), _uniqueTimestampCounter(0) { @@ -252,13 +240,11 @@ TestDistributorApp::TestDistributorApp(vespalib::stringref configId) configure(configId); } -TestDistributorApp::TestDistributorApp(NodeIndex index, - vespalib::stringref configId) +TestDistributorApp::TestDistributorApp(NodeIndex index, vespalib::stringref configId) : TestStorageApp( std::make_unique<DistributorComponentRegisterImpl>(), lib::NodeType::DISTRIBUTOR, index, configId), - _compReg(dynamic_cast<DistributorComponentRegisterImpl&>( - TestStorageApp::getComponentRegister())), + _compReg(dynamic_cast<DistributorComponentRegisterImpl&>(TestStorageApp::getComponentRegister())), _lastUniqueTimestampRequested(0), _uniqueTimestampCounter(0) { @@ -266,12 +252,12 @@ TestDistributorApp::TestDistributorApp(NodeIndex index, configure(configId); } -TestDistributorApp::~TestDistributorApp() {} +TestDistributorApp::~TestDistributorApp() = default; api::Timestamp TestDistributorApp::getUniqueTimestamp() { - vespalib::Lock lock(_accessLock); + std::lock_guard guard(_accessLock); uint64_t timeNow(getClock().getTimeInSeconds().getTime()); if (timeNow == _lastUniqueTimestampRequested) { ++_uniqueTimestampCounter; diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 218e7352f04..03936d37788 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -141,16 +141,16 @@ class TestDistributorApp : public TestStorageApp, public UniqueTimeCalculator { DistributorComponentRegisterImpl& _compReg; - vespalib::Lock _accessLock; + std::mutex _accessLock; uint64_t _lastUniqueTimestampRequested; uint32_t _uniqueTimestampCounter; void configure(vespalib::stringref configId); public: - TestDistributorApp(vespalib::stringref configId = ""); - TestDistributorApp(NodeIndex index, vespalib::stringref configId = ""); - ~TestDistributorApp(); + explicit TestDistributorApp(vespalib::stringref configId = ""); + explicit TestDistributorApp(NodeIndex index, vespalib::stringref configId = ""); + ~TestDistributorApp() override; DistributorComponentRegisterImpl& getComponentRegister() { return _compReg; diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index b7f2cb81956..e510076fe9b 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -2049,4 +2049,24 @@ TEST_F(FileStorManagerTest, get_command_size_is_added_to_metric) { assert_request_size_set(c, std::move(cmd), thread_metrics_of(*c.manager)->get[defaultLoadType]); } +TEST_F(FileStorManagerTest, test_and_set_condition_mismatch_not_counted_as_failure) { + TestFileStorComponents c(*this); + auto doc = _node->getTestDocMan().createRandomDocument(); + document::BucketId bucket(16, document::BucketIdFactory().getBucketId(doc->getId()).getRawId()); + createBucket(bucket, 0); + auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), std::move(doc), api::Timestamp(12345)); + cmd->setCondition(TestAndSetCondition("not testdoctype1")); + cmd->setAddress(api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 3)); + c.top.sendDown(cmd); + + api::PutReply* reply; + ASSERT_SINGLE_REPLY(api::PutReply, reply, c.top, _waitTime); + EXPECT_EQ(reply->getResult().getResult(), api::ReturnCode::TEST_AND_SET_CONDITION_FAILED); + + auto& metrics = thread_metrics_of(*c.manager)->put[defaultLoadType]; + EXPECT_EQ(metrics.failed.getValue(), 0); + EXPECT_EQ(metrics.test_and_set_failed.getValue(), 1); + EXPECT_EQ(thread_metrics_of(*c.manager)->failedOperations.getValue(), 0); +} + } // storage diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h index 3d25a205017..d4cbd896d11 100644 --- a/storage/src/tests/persistence/persistencetestutils.h +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -42,7 +42,7 @@ public: class NoBucketLock : public FileStorHandler::BucketLockInterface { public: - NoBucketLock(document::Bucket bucket) : _bucket(bucket) { } + NoBucketLock(document::Bucket bucket) noexcept : _bucket(bucket) { } const document::Bucket &getBucket() const override { return _bucket; } diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 178862d8393..6ac0bfcb5d8 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -216,7 +216,7 @@ void waitUntilMergeQueueIs(MergeThrottler& throttler, std::size_t sz, int timeou while (true) { std::size_t count; { - vespalib::LockGuard lock(throttler.getStateLock()); + std::lock_guard lock(throttler.getStateLock()); count = throttler.getMergeQueue().size(); } if (count == sz) { diff --git a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp index 8228ceb5e79..69d5a827272 100644 --- a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp @@ -265,7 +265,7 @@ struct StorageApiRpcServiceTest : Test { return recv_as_put; } [[nodiscard]] std::shared_ptr<api::PutCommand> send_and_receive_put_command_at_node_1() { - return send_and_receive_put_command_at_node_1([]([[maybe_unused]] auto& cmd){}); + return send_and_receive_put_command_at_node_1([]([[maybe_unused]] auto& cmd) noexcept {}); } [[nodiscard]] std::shared_ptr<api::PutReply> respond_and_receive_put_reply_at_node_0( @@ -283,7 +283,7 @@ struct StorageApiRpcServiceTest : Test { [[nodiscard]] std::shared_ptr<api::PutReply> respond_and_receive_put_reply_at_node_0( const std::shared_ptr<api::PutCommand>& cmd) { - return respond_and_receive_put_reply_at_node_0(cmd, []([[maybe_unused]] auto& reply){}); + return respond_and_receive_put_reply_at_node_0(cmd, []([[maybe_unused]] auto& reply) noexcept {}); } }; diff --git a/storage/src/tests/storageserver/testvisitormessagesession.h b/storage/src/tests/storageserver/testvisitormessagesession.h index a9d83ca6c1a..1cea1c78d74 100644 --- a/storage/src/tests/storageserver/testvisitormessagesession.h +++ b/storage/src/tests/storageserver/testvisitormessagesession.h @@ -45,7 +45,7 @@ public: struct TestVisitorMessageSessionFactory : public VisitorMessageSessionFactory { - vespalib::Lock _accessLock; + std::mutex _accessLock; std::vector<TestVisitorMessageSession*> _visitorSessions; mbus::Error _autoReplyError; bool _createAutoReplyVisitorSessions; @@ -56,11 +56,10 @@ struct TestVisitorMessageSessionFactory : public VisitorMessageSessionFactory _priConverter(configId) {} VisitorMessageSession::UP createSession(Visitor& v, VisitorThread& vt) override { - vespalib::LockGuard lock(_accessLock); - TestVisitorMessageSession::UP session(new TestVisitorMessageSession(vt, v, _autoReplyError, - _createAutoReplyVisitorSessions)); + std::lock_guard lock(_accessLock); + auto session = std::make_unique<TestVisitorMessageSession>(vt, v, _autoReplyError, _createAutoReplyVisitorSessions); _visitorSessions.push_back(session.get()); - return VisitorMessageSession::UP(std::move(session)); + return session; } documentapi::Priority::Value toDocumentPriority(uint8_t storagePriority) const override { diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index 20934d04eaa..0ce305fb8bb 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -223,7 +223,7 @@ VisitorManagerTest::getSession(uint32_t n) framework::MilliSecTime endTime(clock.getTimeInMillis() + framework::MilliSecTime(30 * 1000)); while (true) { { - vespalib::LockGuard lock(_messageSessionFactory->_accessLock); + std::lock_guard lock(_messageSessionFactory->_accessLock); if (sessions.size() > n) { return *sessions[n]; } diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 16b2fca77ae..ee6b8fa3d38 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/document/fieldvalue/rawfieldvalue.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> @@ -259,7 +257,7 @@ VisitorTest::getSession(uint32_t n) clock.getTimeInMillis() + framework::MilliSecTime(30 * 1000)); while (true) { { - vespalib::LockGuard lock(_messageSessionFactory->_accessLock); + std::lock_guard lock(_messageSessionFactory->_accessLock); if (sessions.size() > n) { return *sessions[n]; } diff --git a/storage/src/vespa/storage/bucketdb/abstract_bucket_map.h b/storage/src/vespa/storage/bucketdb/abstract_bucket_map.h index 9b738b26994..be83970e2b7 100644 --- a/storage/src/vespa/storage/bucketdb/abstract_bucket_map.h +++ b/storage/src/vespa/storage/bucketdb/abstract_bucket_map.h @@ -36,7 +36,7 @@ public: key_type _key; bool _locked; - LockKeeper(AbstractBucketMap& map, key_type key) + LockKeeper(AbstractBucketMap& map, key_type key) noexcept : _map(map), _key(key), _locked(true) {} void unlock() { _map.unlock(_key); _locked = false; } public: @@ -44,7 +44,7 @@ public: }; struct WrappedEntry { - WrappedEntry() + WrappedEntry() noexcept : _exists(false), _preExisted(false), _lockKeeper(), @@ -103,16 +103,16 @@ public: key_type _key; const char* _owner; - LockId() : _key(0), _owner("none - empty token") {} - LockId(key_type key, const char* owner) + LockId() noexcept : _key(0), _owner("none - empty token") {} + LockId(key_type key, const char* owner) noexcept : _key(key), _owner(owner) { assert(_owner); } - size_t hash() const { return _key; } - size_t operator%(size_t val) const { return _key % val; } - bool operator==(const LockId& id) const { return (_key == id._key); } + size_t hash() const noexcept { return _key; } + size_t operator%(size_t val) const noexcept { return _key % val; } + bool operator==(const LockId& id) const noexcept { return (_key == id._key); } operator key_type() const { return _key; } }; diff --git a/storage/src/vespa/storage/bucketdb/bucketcopy.h b/storage/src/vespa/storage/bucketdb/bucketcopy.h index 09f86ef1324..94a1e63e53e 100644 --- a/storage/src/vespa/storage/bucketdb/bucketcopy.h +++ b/storage/src/vespa/storage/bucketdb/bucketcopy.h @@ -15,12 +15,12 @@ private: public: static const int TRUSTED = 1; - BucketCopy() + BucketCopy() noexcept : _timestamp(0), _flags(0), _node(0xffff) {} BucketCopy(uint64_t timestamp, uint16_t nodeIdx, - const api::BucketInfo& info) + const api::BucketInfo& info) noexcept : _timestamp(timestamp), _info(info), _flags(0), diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp index d11d24c7acf..57183566964 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ b/storage/src/vespa/storage/bucketdb/lockablemap.hpp @@ -354,7 +354,7 @@ LockableMap<Map>::print(std::ostream& out, bool verbose, out << "LockableMap {\n" << indent << " "; if (verbose) { - for (const auto & entry : _map) { + for (const auto entry : _map) { out << "Key: " << BucketId(BucketId::keyToBucketId(entry.first)) << " Value: " << entry.second << "\n" << indent << " "; } diff --git a/storage/src/vespa/storage/common/distributorcomponent.h b/storage/src/vespa/storage/common/distributorcomponent.h index e55e6f45cb1..27680b8c6ce 100644 --- a/storage/src/vespa/storage/common/distributorcomponent.h +++ b/storage/src/vespa/storage/common/distributorcomponent.h @@ -60,8 +60,7 @@ struct DistributorManagedComponent struct DistributorComponentRegister : public virtual StorageComponentRegister { - virtual void registerDistributorComponent( - DistributorManagedComponent&) = 0; + virtual void registerDistributorComponent(DistributorManagedComponent&) = 0; }; class DistributorComponent : public StorageComponent, @@ -86,10 +85,10 @@ public: typedef std::unique_ptr<DistributorComponent> UP; DistributorComponent(DistributorComponentRegister& compReg, vespalib::stringref name); - ~DistributorComponent(); + ~DistributorComponent() override; api::Timestamp getUniqueTimestamp() const { - assert(_timeCalculator); return _timeCalculator->getUniqueTimestamp(); + return _timeCalculator->getUniqueTimestamp(); } const DistributorConfig& getDistributorConfig() const { return _distributorConfig; diff --git a/storage/src/vespa/storage/common/servicelayercomponent.h b/storage/src/vespa/storage/common/servicelayercomponent.h index 22a33aac14c..2308480411e 100644 --- a/storage/src/vespa/storage/common/servicelayercomponent.h +++ b/storage/src/vespa/storage/common/servicelayercomponent.h @@ -79,11 +79,9 @@ public: const ContentBucketSpaceRepo &getBucketSpaceRepo() const; StorBucketDatabase& getBucketDatabase(document::BucketSpace bucketSpace) const; MinimumUsedBitsTracker& getMinUsedBitsTracker() { - assert(_minUsedBitsTracker != 0); return *_minUsedBitsTracker; } const MinimumUsedBitsTracker& getMinUsedBitsTracker() const { - assert(_minUsedBitsTracker != 0); return *_minUsedBitsTracker; } uint16_t getIdealPartition(const document::Bucket&) const; diff --git a/storage/src/vespa/storage/common/storagelinkqueued.hpp b/storage/src/vespa/storage/common/storagelinkqueued.hpp index 22c5e9ba5f2..ea6430f9e5c 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.hpp +++ b/storage/src/vespa/storage/common/storagelinkqueued.hpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <sstream> #include <chrono> +#include <cassert> namespace storage { diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index cfd8d7f1753..352da70d7e4 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -28,7 +28,7 @@ class Distributor::Status { bool _done; public: - Status(const DelegatedStatusRequest& request) + Status(const DelegatedStatusRequest& request) noexcept : _request(request), _monitor(), _done(false) @@ -465,7 +465,7 @@ BucketSpacesStatsProvider::BucketSpacesStats Distributor::make_invalid_stats_per } void Distributor::invalidate_bucket_spaces_stats() { - vespalib::LockGuard guard(_metricLock); + std::lock_guard guard(_metricLock); _bucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats(); auto invalid_space_stats = make_invalid_stats_per_configured_space(); @@ -681,21 +681,21 @@ void Distributor::startExternalOperations() { std::unordered_map<uint16_t, uint32_t> Distributor::getMinReplica() const { - vespalib::LockGuard guard(_metricLock); + std::lock_guard guard(_metricLock); return _bucketDbStats._minBucketReplica; } BucketSpacesStatsProvider::PerNodeBucketSpacesStats Distributor::getBucketSpacesStats() const { - vespalib::LockGuard guard(_metricLock); + std::lock_guard guard(_metricLock); return _bucketSpacesStats; } void Distributor::propagateInternalScanMetricsToExternal() { - vespalib::LockGuard guard(_metricLock); + std::lock_guard guard(_metricLock); // All shared values are written when _metricLock is held, so no races. if (_bucketDBMetricUpdater.hasCompletedRound()) { @@ -755,7 +755,7 @@ bool merge_no_longer_pending_edge(const PerNodeBucketSpacesStats& prev_stats, void Distributor::updateInternalMetricsForCompletedScan() { - vespalib::LockGuard guard(_metricLock); + std::lock_guard guard(_metricLock); _bucketDBMetricUpdater.completeRound(); _bucketDbStats = _bucketDBMetricUpdater.getLastCompleteStats(); diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index fcc08030764..d7dd1fda2e9 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -20,7 +20,6 @@ #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageframework/generic/thread/tickingthread.h> -#include <vespa/vespalib/util/sync.h> #include <queue> #include <unordered_map> @@ -301,7 +300,6 @@ private: ClientRequestPriorityQueue _client_request_priority_queue; MessageQueue _fetchedMessages; framework::TickingThreadPool& _threadPool; - vespalib::Monitor _statusMonitor; mutable std::vector<std::shared_ptr<Status>> _statusToDo; mutable std::vector<std::shared_ptr<Status>> _fetchedStatusRequests; @@ -325,7 +323,7 @@ private: BucketDBMetricUpdater _bucketDBMetricUpdater; std::unique_ptr<BucketGcTimeCalculator::BucketIdHasher> _bucketIdHasher; MetricUpdateHook _metricUpdateHook; - vespalib::Lock _metricLock; + mutable std::mutex _metricLock; /** * Maintenance stats for last completed database scan iteration. * Access must be protected by _metricLock as it is read by metric diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.h b/storage/src/vespa/storage/distributor/distributorcomponent.h index df25efb3b00..283d0c20390 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.h +++ b/storage/src/vespa/storage/distributor/distributorcomponent.h @@ -186,7 +186,6 @@ protected: DistributorBucketSpaceRepo& _bucketSpaceRepo; DistributorBucketSpaceRepo& _readOnlyBucketSpaceRepo; - vespalib::Lock _sync; }; } diff --git a/storage/src/vespa/storage/distributor/messageguard.h b/storage/src/vespa/storage/distributor/messageguard.h index 9c71758a24b..b2be0cc67ff 100644 --- a/storage/src/vespa/storage/distributor/messageguard.h +++ b/storage/src/vespa/storage/distributor/messageguard.h @@ -3,7 +3,6 @@ #include "pendingclusterstate.h" #include <vespa/storage/common/messagesender.h> -#include <vespa/vespalib/util/sync.h> namespace storage { @@ -11,12 +10,11 @@ class MessageGuard { std::vector<std::shared_ptr<api::StorageMessage> > messagesUp; std::vector<std::shared_ptr<api::StorageMessage> > messagesDown; - vespalib::LockGuard _lock; + std::unique_lock<std::mutex> _lock; ChainedMessageSender& _messageSender; public: - MessageGuard(const vespalib::Lock &lock, - ChainedMessageSender& messageSender) + MessageGuard(std::mutex & lock, ChainedMessageSender& messageSender) : _lock(lock), _messageSender(messageSender) {} diff --git a/storage/src/vespa/storage/distributor/messagetracker.h b/storage/src/vespa/storage/distributor/messagetracker.h index 75ae287d98f..11d8c36082e 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.h +++ b/storage/src/vespa/storage/distributor/messagetracker.h @@ -17,7 +17,7 @@ class MessageTracker { public: class ToSend { public: - ToSend(std::shared_ptr<api::BucketCommand> msg, uint16_t target) : + ToSend(std::shared_ptr<api::BucketCommand> msg, uint16_t target) noexcept : _msg(std::move(msg)), _target(target) {}; std::shared_ptr<api::BucketCommand> _msg; diff --git a/storage/src/vespa/storage/distributor/operation_sequencer.h b/storage/src/vespa/storage/distributor/operation_sequencer.h index cf687e721a8..7a523b61e53 100644 --- a/storage/src/vespa/storage/distributor/operation_sequencer.h +++ b/storage/src/vespa/storage/distributor/operation_sequencer.h @@ -24,7 +24,7 @@ class SequencingHandle { OperationSequencer* _sequencer; document::GlobalId _gid; public: - SequencingHandle() : _sequencer(nullptr) {} + SequencingHandle() noexcept : _sequencer(nullptr) {} SequencingHandle(OperationSequencer& sequencer, const document::GlobalId& gid) : _sequencer(&sequencer), _gid(gid) diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h index c977ea78f67..aa38a0d2319 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h @@ -4,7 +4,6 @@ #include <vespa/vespalib/util/hdr_abort.h> #include <vespa/storage/distributor/operations/operation.h> #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <vespa/vespalib/util/sync.h> namespace storage { diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.h b/storage/src/vespa/storage/distributor/operations/external/updateoperation.h index c66026f02f5..9d9bce9160b 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.h @@ -55,7 +55,7 @@ private: class PreviousDocumentVersion { public: - PreviousDocumentVersion(document::BucketId b, const api::BucketInfo& info, uint64_t o, uint16_t node) : + PreviousDocumentVersion(document::BucketId b, const api::BucketInfo& info, uint64_t o, uint16_t node) noexcept : bucketId(b), bucketInfo(info), oldTs(o), nodeId(node) {} document::BucketId bucketId; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h index fb84441b654..273603ca6ad 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h @@ -3,26 +3,22 @@ #pragma once #include <vespa/storage/bucketdb/bucketcopy.h> -#include <cassert> -namespace storage { -namespace distributor { +namespace storage::distributor { struct MergeMetaData { uint16_t _nodeIndex; bool _sourceOnly; const BucketCopy* _copy; - MergeMetaData() : _nodeIndex(0), _sourceOnly(false), _copy(0) {} - MergeMetaData(uint16_t nodeIndex, const BucketCopy& copy) + MergeMetaData() noexcept : _nodeIndex(0), _sourceOnly(false), _copy(nullptr) {} + MergeMetaData(uint16_t nodeIndex, const BucketCopy& copy) noexcept : _nodeIndex(nodeIndex), _sourceOnly(false), _copy(©) {} bool trusted() const { - assert(_copy != 0); return _copy->trusted(); } uint32_t checksum() const { - assert(_copy != 0); return _copy->getChecksum(); } bool source_only() const noexcept { return _sourceOnly; } @@ -30,6 +26,4 @@ struct MergeMetaData { vespalib::asciistream& operator<<(vespalib::asciistream& out, const MergeMetaData& e); -} // distributor -} // storage - +} diff --git a/storage/src/vespa/storage/distributor/sentmessagemap.h b/storage/src/vespa/storage/distributor/sentmessagemap.h index 405e59f7b31..e84920b88d7 100644 --- a/storage/src/vespa/storage/distributor/sentmessagemap.h +++ b/storage/src/vespa/storage/distributor/sentmessagemap.h @@ -1,14 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/sync.h> #include <map> #include <vespa/storageapi/messageapi/storagemessage.h> -namespace storage -{ - -namespace distributor { +namespace storage::distributor { class Operation; @@ -32,6 +28,3 @@ private: }; } - -} - diff --git a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.cpp index 439bc9e078c..efd558a0b4b 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "distributorcomponentregisterimpl.h" -#include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vdslib/state/cluster_state_bundle.h> @@ -11,8 +10,7 @@ DistributorComponentRegisterImpl::DistributorComponentRegisterImpl() { } -DistributorComponentRegisterImpl::~DistributorComponentRegisterImpl() { -} +DistributorComponentRegisterImpl::~DistributorComponentRegisterImpl() = default; void DistributorComponentRegisterImpl::handleNewState() @@ -24,7 +22,7 @@ DistributorComponentRegisterImpl::handleNewState() void DistributorComponentRegisterImpl::registerDistributorComponent(DistributorManagedComponent& smc) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _components.push_back(&smc); if (_timeCalculator != 0) { smc.setTimeCalculator(*_timeCalculator); @@ -36,7 +34,7 @@ DistributorComponentRegisterImpl::registerDistributorComponent(DistributorManage void DistributorComponentRegisterImpl::setTimeCalculator(UniqueTimeCalculator& utc) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); if (_timeCalculator != 0) { throw vespalib::IllegalStateException( "Time calculator already set. Cannot be updated live", @@ -51,7 +49,7 @@ DistributorComponentRegisterImpl::setTimeCalculator(UniqueTimeCalculator& utc) void DistributorComponentRegisterImpl::setDistributorConfig(const DistributorConfig& c) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _distributorConfig = c; for (uint32_t i=0; i<_components.size(); ++i) { _components[i]->setDistributorConfig(c); @@ -61,7 +59,7 @@ DistributorComponentRegisterImpl::setDistributorConfig(const DistributorConfig& void DistributorComponentRegisterImpl::setVisitorConfig(const VisitorConfig& c) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _visitorConfig = c; for (uint32_t i=0; i<_components.size(); ++i) { _components[i]->setVisitorConfig(c); diff --git a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h index 61fba7f9921..a6c678630e0 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h @@ -21,7 +21,7 @@ class DistributorComponentRegisterImpl public virtual StorageComponentRegisterImpl, private StateListener { - vespalib::Lock _componentLock; + std::mutex _componentLock; std::vector<DistributorManagedComponent*> _components; UniqueTimeCalculator* _timeCalculator; diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp index da45b61701e..17e65478411 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp @@ -18,7 +18,7 @@ void ServiceLayerComponentRegisterImpl::registerServiceLayerComponent( ServiceLayerManagedComponent& smc) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _components.push_back(&smc); smc.setDiskCount(_diskCount); smc.setBucketSpaceRepo(_bucketSpaceRepo); @@ -28,7 +28,7 @@ ServiceLayerComponentRegisterImpl::registerServiceLayerComponent( void ServiceLayerComponentRegisterImpl::setDiskCount(uint16_t count) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); if (_diskCount != 0) { throw IllegalStateException("Disk count already set. Cannot be updated live", VESPA_STRLOC); } diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index 9988b861422..e722110d770 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -18,7 +18,7 @@ class ServiceLayerComponentRegisterImpl : public virtual ServiceLayerComponentRegister, public virtual StorageComponentRegisterImpl { - vespalib::Lock _componentLock; + std::mutex _componentLock; std::vector<ServiceLayerManagedComponent*> _components; uint16_t _diskCount; ContentBucketSpaceRepo _bucketSpaceRepo; diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp index 654fcbd1380..9c421eb7f8f 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp @@ -2,9 +2,9 @@ #include "storagecomponentregisterimpl.h" #include <vespa/vespalib/util/exceptions.h> +#include <cassert> #include <vespa/log/log.h> - LOG_SETUP(".storage.component.register"); namespace storage { @@ -30,7 +30,7 @@ StorageComponentRegisterImpl::~StorageComponentRegisterImpl() = default; void StorageComponentRegisterImpl::registerStorageComponent(StorageComponent& smc) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _components.push_back(&smc); assert(_nodeType != nullptr); smc.setNodeInfo(_clusterName, *_nodeType, _index); @@ -49,7 +49,7 @@ StorageComponentRegisterImpl::setNodeInfo(vespalib::stringref clusterName, const lib::NodeType& nodeType, uint16_t index) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); if (_nodeType != nullptr) { LOG(warning, "Node info already set. May be valid in tests, but is a " "bug in production. Node info should not be updated live"); @@ -62,7 +62,7 @@ StorageComponentRegisterImpl::setNodeInfo(vespalib::stringref clusterName, void StorageComponentRegisterImpl::setNodeStateUpdater(NodeStateUpdater& updater) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); if (_nodeStateUpdater != 0) { throw vespalib::IllegalStateException( "Node state updater already set. Should never be altered live.", @@ -77,7 +77,7 @@ StorageComponentRegisterImpl::setNodeStateUpdater(NodeStateUpdater& updater) void StorageComponentRegisterImpl::setDocumentTypeRepo(std::shared_ptr<const document::DocumentTypeRepo> repo) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _docTypeRepo = repo; for (auto& component : _components) { component->setDocumentTypeRepo(repo); @@ -87,7 +87,7 @@ StorageComponentRegisterImpl::setDocumentTypeRepo(std::shared_ptr<const document void StorageComponentRegisterImpl::setLoadTypes(documentapi::LoadTypeSet::SP loadTypes) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _loadTypes = loadTypes; for (auto& component : _components) { component->setLoadTypes(loadTypes); @@ -97,7 +97,7 @@ StorageComponentRegisterImpl::setLoadTypes(documentapi::LoadTypeSet::SP loadType void StorageComponentRegisterImpl::setPriorityConfig(const PriorityConfig& config) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _priorityConfig = config; for (auto& component : _components) { component->setPriorityConfig(config); @@ -107,7 +107,7 @@ StorageComponentRegisterImpl::setPriorityConfig(const PriorityConfig& config) void StorageComponentRegisterImpl::setBucketIdFactory(const document::BucketIdFactory& factory) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _bucketIdFactory = factory; for (auto& component : _components) { component->setBucketIdFactory(factory); @@ -117,7 +117,7 @@ StorageComponentRegisterImpl::setBucketIdFactory(const document::BucketIdFactory void StorageComponentRegisterImpl::setDistribution(lib::Distribution::SP distribution) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _distribution = distribution; for (auto& component : _components) { component->setDistribution(distribution); @@ -127,7 +127,7 @@ StorageComponentRegisterImpl::setDistribution(lib::Distribution::SP distribution void StorageComponentRegisterImpl::setBucketSpacesConfig(const BucketspacesConfig& config) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _bucketSpacesConfig = config; } diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h index 2f8bfaab87b..b22ea93223b 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h @@ -24,7 +24,7 @@ class StorageComponentRegisterImpl using PriorityConfig = StorageComponent::PriorityConfig; using BucketspacesConfig = vespa::config::content::core::internal::InternalBucketspacesType; - vespalib::Lock _componentLock; + std::mutex _componentLock; std::vector<StorageComponent*> _components; vespalib::string _clusterName; const lib::NodeType* _nodeType; @@ -41,24 +41,20 @@ public: typedef std::unique_ptr<StorageComponentRegisterImpl> UP; StorageComponentRegisterImpl(); - ~StorageComponentRegisterImpl(); + ~StorageComponentRegisterImpl() override; const vespalib::string& getClusterName() const { return _clusterName; } - const lib::NodeType& getNodeType() const - { assert(_nodeType != 0); return *_nodeType; } + const lib::NodeType& getNodeType() const { return *_nodeType; } uint16_t getIndex() const { return _index; } std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() { return _docTypeRepo; } documentapi::LoadTypeSet::SP getLoadTypes() { return _loadTypes; } const document::BucketIdFactory& getBucketIdFactory() { return _bucketIdFactory; } lib::Distribution::SP getDistribution() { return _distribution; } - NodeStateUpdater& getNodeStateUpdater() - { assert(_nodeStateUpdater != 0); return *_nodeStateUpdater; } + NodeStateUpdater& getNodeStateUpdater() { return *_nodeStateUpdater; } void registerStorageComponent(StorageComponent&) override; - void setNodeInfo(vespalib::stringref clusterName, - const lib::NodeType& nodeType, - uint16_t index); + void setNodeInfo(vespalib::stringref clusterName, const lib::NodeType& nodeType, uint16_t index); virtual void setNodeStateUpdater(NodeStateUpdater& updater); virtual void setDocumentTypeRepo(std::shared_ptr<const document::DocumentTypeRepo>); virtual void setLoadTypes(documentapi::LoadTypeSet::SP); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 4df2b5e591b..75348a25e76 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -75,8 +75,8 @@ FileStorHandlerImpl::~FileStorHandlerImpl() = default; void FileStorHandlerImpl::addMergeStatus(const document::Bucket& bucket, MergeStatus::SP status) { - vespalib::LockGuard mlock(_mergeStatesLock); - if (_mergeStates.find(bucket) != _mergeStates.end()) {; + std::lock_guard mlock(_mergeStatesLock); + if (_mergeStates.find(bucket) != _mergeStates.end()) { LOG(warning, "A merge status already existed for %s. Overwriting it.", bucket.toString().c_str()); } _mergeStates[bucket] = status; @@ -85,7 +85,7 @@ FileStorHandlerImpl::addMergeStatus(const document::Bucket& bucket, MergeStatus: MergeStatus& FileStorHandlerImpl::editMergeStatus(const document::Bucket& bucket) { - vespalib::LockGuard mlock(_mergeStatesLock); + std::lock_guard mlock(_mergeStatesLock); MergeStatus::SP status = _mergeStates[bucket]; if (status.get() == 0) { throw vespalib::IllegalStateException("No merge state exist for " + bucket.toString(), VESPA_STRLOC); @@ -96,21 +96,21 @@ FileStorHandlerImpl::editMergeStatus(const document::Bucket& bucket) bool FileStorHandlerImpl::isMerging(const document::Bucket& bucket) const { - vespalib::LockGuard mlock(_mergeStatesLock); + std::lock_guard mlock(_mergeStatesLock); return (_mergeStates.find(bucket) != _mergeStates.end()); } uint32_t FileStorHandlerImpl::getNumActiveMerges() const { - vespalib::LockGuard mlock(_mergeStatesLock); + std::lock_guard mlock(_mergeStatesLock); return _mergeStates.size(); } void FileStorHandlerImpl::clearMergeStatus(const document::Bucket& bucket, const api::ReturnCode* code) { - vespalib::LockGuard mlock(_mergeStatesLock); + std::lock_guard mlock(_mergeStatesLock); auto it = _mergeStates.find(bucket); if (it == _mergeStates.end()) { if (code != 0) { @@ -301,7 +301,7 @@ void FileStorHandlerImpl::updateMetrics(const MetricLockGuard &) { for (Disk & disk : _diskInfo) { - vespalib::MonitorGuard lockGuard(_mergeStatesLock); + std::lock_guard lockGuard(_mergeStatesLock); disk.metrics->pendingMerges.addValue(_mergeStates.size()); disk.metrics->queueSize.addValue(disk.getQueueSize()); @@ -802,7 +802,7 @@ FileStorHandlerImpl::Stripe::failOperations(const document::Bucket &bucket, cons { vespalib::MonitorGuard guard(_lock); - BucketIdx& idx(bmi::get<2>(_queue)); + BucketIdx& idx(bmi::get<2>(*_queue)); std::pair<BucketIdx::iterator, BucketIdx::iterator> range(idx.equal_range(bucket)); for (auto iter = range.first; iter != range.second;) { @@ -869,9 +869,13 @@ FileStorHandlerImpl::MessageEntry::~MessageEntry() = default; FileStorHandlerImpl::Disk::Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numStripes) : metrics(0), _nextStripeId(0), - _stripes(numStripes, Stripe(owner, messageSender)), + _stripes(), state(FileStorHandler::AVAILABLE) { + _stripes.reserve(numStripes); + for (size_t i(0); i < numStripes; i++) { + _stripes.emplace_back(owner, messageSender); + } assert(numStripes > 0); } @@ -884,6 +888,7 @@ FileStorHandlerImpl::Disk::Disk(Disk && rhs) noexcept FileStorHandlerImpl::Disk::~Disk() = default; FileStorHandlerImpl::Stripe::~Stripe() = default; +FileStorHandlerImpl::Stripe::Stripe(Stripe &&) noexcept = default; void FileStorHandlerImpl::Disk::flush() @@ -903,7 +908,7 @@ FileStorHandlerImpl::Disk::broadcast() uint64_t FileStorHandlerImpl::Disk::dispersed_bucket_bits(const document::Bucket& bucket) noexcept { const uint64_t raw_id = bucket.getBucketId().getId(); - return XXH64(&raw_id, sizeof(uint64_t), 0); + return XXH3_64bits(&raw_id, sizeof(uint64_t)); } bool @@ -921,6 +926,10 @@ FileStorHandlerImpl::Disk::schedule(const std::shared_ptr<api::StorageMessage>& FileStorHandlerImpl::Stripe::Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender) : _owner(owner), _messageSender(messageSender), + _metrics(nullptr), + _lock(), + _queue(std::make_unique<PriorityQueue>()), + _lockedBuckets(), _active_merges(0) {} @@ -933,7 +942,7 @@ FileStorHandlerImpl::Stripe::getNextMessage(uint32_t timeout, Disk & disk) // second attempt. This is key to allowing the run loop to register // ticks at regular intervals while not busy-waiting. for (int attempt = 0; (attempt < 2) && ! disk.isClosed() && !_owner.isPaused(); ++attempt) { - PriorityIdx& idx(bmi::get<1>(_queue)); + PriorityIdx& idx(bmi::get<1>(*_queue)); PriorityIdx::iterator iter(idx.begin()), end(idx.end()); while (iter != end && operationIsInhibited(guard, iter->_bucket, *iter->_command)) { @@ -1032,11 +1041,11 @@ void FileStorHandlerImpl::Stripe::abort(std::vector<std::shared_ptr<api::Storage const AbortBucketOperationsCommand& cmd) { vespalib::MonitorGuard lockGuard(_lock); - for (auto it(_queue.begin()); it != _queue.end();) { + for (auto it(_queue->begin()); it != _queue->end();) { api::StorageMessage& msg(*it->_command); if (messageMayBeAborted(msg) && cmd.shouldAbort(it->_bucket)) { aborted.emplace_back(static_cast<api::StorageCommand&>(msg).makeReply()); - it = _queue.erase(it); + it = _queue->erase(it); } else { ++it; } @@ -1046,7 +1055,7 @@ void FileStorHandlerImpl::Stripe::abort(std::vector<std::shared_ptr<api::Storage bool FileStorHandlerImpl::Stripe::schedule(MessageEntry messageEntry) { vespalib::MonitorGuard lockGuard(_lock); - _queue.emplace_back(std::move(messageEntry)); + _queue->emplace_back(std::move(messageEntry)); lockGuard.broadcast(); return true; } @@ -1055,8 +1064,8 @@ void FileStorHandlerImpl::Stripe::flush() { vespalib::MonitorGuard lockGuard(_lock); - while (!(_queue.empty() && _lockedBuckets.empty())) { - LOG(debug, "Still %ld in queue and %ld locked buckets", _queue.size(), _lockedBuckets.size()); + while (!(_queue->empty() && _lockedBuckets.empty())) { + LOG(debug, "Still %ld in queue and %ld locked buckets", _queue->size(), _lockedBuckets.size()); lockGuard.wait(100); } } @@ -1229,7 +1238,7 @@ FileStorHandlerImpl::Stripe::dumpQueueHtml(std::ostream & os) const { vespalib::MonitorGuard guard(_lock); - const PriorityIdx& idx = bmi::get<1>(_queue); + const PriorityIdx& idx = bmi::get<1>(*_queue); for (const auto & entry : idx) { os << "<li>" << entry._command->toString() << " (priority: " << (int)entry._command->getPriority() << ")</li>\n"; @@ -1238,8 +1247,9 @@ FileStorHandlerImpl::Stripe::dumpQueueHtml(std::ostream & os) const namespace { -void dump_lock_entry(const document::BucketId& bucketId, const FileStorHandlerImpl::Stripe::LockEntry& entry, - api::LockingRequirements lock_mode, FileStorHandlerImpl::Clock::time_point now_ts, std::ostream& os) { +void +dump_lock_entry(const document::BucketId& bucketId, const FileStorHandlerImpl::Stripe::LockEntry& entry, + api::LockingRequirements lock_mode, FileStorHandlerImpl::Clock::time_point now_ts, std::ostream& os) { os << api::MessageType::get(entry.msgType).getName() << ":" << entry.msgId << " (" << bucketId << ", " << api::to_string(lock_mode) << " lock) Running for " << std::chrono::duration_cast<std::chrono::seconds>(now_ts - entry.timestamp).count() << " secs<br/>\n"; @@ -1269,7 +1279,7 @@ FileStorHandlerImpl::Stripe::dumpQueue(std::ostream & os) const { vespalib::MonitorGuard guard(_lock); - const PriorityIdx& idx = bmi::get<1>(_queue); + const PriorityIdx& idx = bmi::get<1>(*_queue); for (const auto & entry : idx) { os << entry._bucket.getBucketId() << ": " << entry._command->toString() << " (priority: " << (int)entry._command->getPriority() << ")\n"; @@ -1300,7 +1310,7 @@ FileStorHandlerImpl::getStatus(std::ostream& out, const framework::HttpUrlPath& out << "</ul>\n"; } - vespalib::LockGuard mergeGuard(_mergeStatesLock); + std::lock_guard mergeGuard(_mergeStatesLock); out << "<tr><td>Active merge operations</td><td>" << _mergeStates.size() << "</td></tr>\n"; if (verbose) { out << "<h4>Active merges</h4>\n"; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index a609473afd5..02dcfea0654 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -84,9 +84,9 @@ public: api::MessageType::Id msgType; api::StorageMessage::Id msgId; - LockEntry() : timestamp(), priority(0), msgType(), msgId(0) { } + LockEntry() noexcept : timestamp(), priority(0), msgType(), msgId(0) { } - LockEntry(uint8_t priority_, api::MessageType::Id msgType_, api::StorageMessage::Id msgId_) + LockEntry(uint8_t priority_, api::MessageType::Id msgType_, api::StorageMessage::Id msgId_) noexcept : timestamp(Clock::now()), priority(priority_), msgType(msgType_), msgId(msgId_) { } }; @@ -98,6 +98,9 @@ public: }; Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender); + Stripe(Stripe &&) noexcept; + Stripe(const Stripe &) = delete; + Stripe & operator =(const Stripe &) = delete; ~Stripe(); void flush(); bool schedule(MessageEntry messageEntry); @@ -111,7 +114,7 @@ public: } size_t getQueueSize() const { vespalib::MonitorGuard guard(_lock); - return _queue.size(); + return _queue->size(); } void release(const document::Bucket & bucket, api::LockingRequirements reqOfReleasedLock, api::StorageMessage::Id lockMsgId); @@ -133,8 +136,8 @@ public: void dumpActiveHtml(std::ostream & os) const; void dumpQueueHtml(std::ostream & os) const; vespalib::Monitor & exposeLock() { return _lock; } - PriorityQueue & exposeQueue() { return _queue; } - BucketIdx & exposeBucketIdx() { return bmi::get<2>(_queue); } + PriorityQueue & exposeQueue() { return *_queue; } + BucketIdx & exposeBucketIdx() { return bmi::get<2>(*_queue); } void setMetrics(FileStorStripeMetrics * metrics) { _metrics = metrics; } private: bool hasActive(vespalib::MonitorGuard & monitor, const AbortBucketOperationsCommand& cmd) const; @@ -143,13 +146,13 @@ public: FileStorHandler::LockedMessage getMessage(vespalib::MonitorGuard & guard, PriorityIdx & idx, PriorityIdx::iterator iter); using LockedBuckets = vespalib::hash_map<document::Bucket, MultiLockEntry, document::Bucket::hash>; - const FileStorHandlerImpl &_owner; - MessageSender &_messageSender; - FileStorStripeMetrics *_metrics; - vespalib::Monitor _lock; - PriorityQueue _queue; - LockedBuckets _lockedBuckets; - uint32_t _active_merges; + const FileStorHandlerImpl &_owner; + MessageSender &_messageSender; + FileStorStripeMetrics *_metrics; + vespalib::Monitor _lock; + std::unique_ptr<PriorityQueue> _queue; + LockedBuckets _lockedBuckets; + uint32_t _active_merges; }; struct Disk { FileStorDiskMetrics * metrics; @@ -207,9 +210,9 @@ public: } std::vector<Stripe> & getStripes() { return _stripes; } private: - uint32_t _nextStripeId; - std::vector<Stripe> _stripes; - std::atomic<DiskState> state; + uint32_t _nextStripeId; + std::vector<Stripe> _stripes; + std::atomic<DiskState> state; }; class BucketLock : public FileStorHandler::BucketLockInterface { @@ -224,9 +227,9 @@ public: api::LockingRequirements lockingRequirements() const noexcept override { return _lockReq; } private: - Stripe & _stripe; - document::Bucket _bucket; - api::StorageMessage::Id _uniqueMsgId; + Stripe & _stripe; + document::Bucket _bucket; + api::StorageMessage::Id _uniqueMsgId; api::LockingRequirements _lockReq; }; @@ -285,7 +288,7 @@ private: std::vector<Disk> _diskInfo; MessageSender& _messageSender; const document::BucketIdFactory& _bucketIdFactory; - vespalib::Lock _mergeStatesLock; + mutable std::mutex _mergeStatesLock; std::map<document::Bucket, MergeStatus::SP> _mergeStates; uint32_t _getNextMessageTimeout; const uint32_t _max_active_merges_per_stripe; // Read concurrently by stripes. diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 85604299e85..0341e34d39d 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -871,7 +871,7 @@ FileStorManager::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPat namespace { struct Deactivator { - StorBucketDatabase::Decision operator()(document::BucketId::Type, StorBucketDatabase::Entry& data) + StorBucketDatabase::Decision operator() (document::BucketId::Type, StorBucketDatabase::Entry& data) noexcept { data.info.setActive(false); return StorBucketDatabase::Decision::UPDATE; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index a80b17d4f19..8e09291a507 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -57,6 +57,34 @@ FileStorThreadMetrics::OpWithRequestSize<BaseOp>::clone( ->assignValues(*this)); } +template <typename BaseOp> +FileStorThreadMetrics::OpWithTestAndSetFailed<BaseOp>::OpWithTestAndSetFailed(const std::string& id, const std::string& name, MetricSet* owner) + : BaseOp(id, name, owner), + test_and_set_failed("test_and_set_failed", {{"yamasdefault"}}, + "Number of times operations were failed due to a " + "test-and-set condition mismatch", this) +{ +} + +template <typename BaseOp> +FileStorThreadMetrics::OpWithTestAndSetFailed<BaseOp>::~OpWithTestAndSetFailed() = default; + +// FIXME this has very non-intuitive semantics, ending up with copy&paste patterns (yet again...) +template <typename BaseOp> +MetricSet* +FileStorThreadMetrics::OpWithTestAndSetFailed<BaseOp>::clone( + std::vector<Metric::UP>& ownerList, + CopyType copyType, + MetricSet* owner, + bool includeUnused) const +{ + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return static_cast<OpWithTestAndSetFailed<BaseOp>*>((new OpWithTestAndSetFailed<BaseOp>(this->getName(), this->_name, owner)) + ->assignValues(*this)); +} + FileStorThreadMetrics::OpWithNotFound::OpWithNotFound(const std::string& id, const std::string& name, MetricSet* owner) : Op(id, name, owner), notFound("not_found", {}, @@ -81,7 +109,7 @@ FileStorThreadMetrics::OpWithNotFound::clone(std::vector<Metric::UP>& ownerList, } FileStorThreadMetrics::Update::Update(MetricSet* owner) - : OpWithRequestSize("update", "Update", owner), + : OpWithTestAndSetFailed("update", "Update", owner), latencyRead("latency_read", {}, "Latency of the source read in the request.", this) { } @@ -122,9 +150,9 @@ FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std: : MetricSet(name, {{"filestor"},{"partofsum"}}, desc), operations("operations", {}, "Number of operations processed.", this), failedOperations("failedoperations", {}, "Number of operations throwing exceptions.", this), - put(lt, OpWithRequestSize<Op>("put", "Put"), this), - get(lt, OpWithRequestSize<OpWithNotFound>("get", "Get"), this), - remove(lt, OpWithRequestSize<OpWithNotFound>("remove", "Remove"), this), + put(lt, PutMetricType("put", "Put"), this), + get(lt, GetMetricType("get", "Get"), this), + remove(lt, RemoveMetricType("remove", "Remove"), this), removeLocation(lt, Op("remove_location", "Remove location"), this), statBucket(lt, Op("stat_bucket", "Stat bucket"), this), update(lt, Update(), this), @@ -258,8 +286,9 @@ template class metrics::LoadMetric<storage::FileStorThreadMetrics::Op>; template class metrics::LoadMetric<storage::FileStorThreadMetrics::OpWithNotFound>; template class metrics::LoadMetric<storage::FileStorThreadMetrics::Update>; template class metrics::LoadMetric<storage::FileStorThreadMetrics::Visitor>; -template class metrics::LoadMetric<storage::FileStorThreadMetrics::OpWithRequestSize<storage::FileStorThreadMetrics::Op>>; -template class metrics::LoadMetric<storage::FileStorThreadMetrics::OpWithRequestSize<storage::FileStorThreadMetrics::OpWithNotFound>>; +template class metrics::LoadMetric<storage::FileStorThreadMetrics::PutMetricType>; +template class metrics::LoadMetric<storage::FileStorThreadMetrics::GetMetricType>; +template class metrics::LoadMetric<storage::FileStorThreadMetrics::RemoveMetricType>; template class metrics::SumMetric<storage::FileStorThreadMetrics::Op>; template class metrics::SumMetric<storage::FileStorThreadMetrics::OpWithNotFound>; template class metrics::SumMetric<storage::FileStorThreadMetrics::Update>; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index be1c5c48213..1ee47495502 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -44,6 +44,17 @@ struct FileStorThreadMetrics : public metrics::MetricSet MetricSet* owner, bool includeUnused) const override; }; + template <typename BaseOp> + struct OpWithTestAndSetFailed : BaseOp { + metrics::LongCountMetric test_and_set_failed; + + OpWithTestAndSetFailed(const std::string& id, const std::string& name, MetricSet* owner = nullptr); + ~OpWithTestAndSetFailed() override; + + MetricSet * clone(std::vector<Metric::UP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const override; + }; + struct OpWithNotFound : Op { metrics::LongCountMetric notFound; @@ -53,7 +64,7 @@ struct FileStorThreadMetrics : public metrics::MetricSet MetricSet* owner, bool includeUnused) const override; }; - struct Update : OpWithRequestSize<OpWithNotFound> { + struct Update : OpWithTestAndSetFailed<OpWithRequestSize<OpWithNotFound>> { metrics::LongAverageMetric latencyRead; explicit Update(MetricSet* owner = nullptr); @@ -73,11 +84,16 @@ struct FileStorThreadMetrics : public metrics::MetricSet MetricSet* owner, bool includeUnused) const override; }; + // FIXME this daisy-chaining approach to metric set variants is not the prettiest... + using PutMetricType = OpWithTestAndSetFailed<OpWithRequestSize<Op>>; + using GetMetricType = OpWithRequestSize<OpWithNotFound>; + using RemoveMetricType = OpWithTestAndSetFailed<OpWithRequestSize<OpWithNotFound>>; + metrics::LongCountMetric operations; metrics::LongCountMetric failedOperations; - metrics::LoadMetric<OpWithRequestSize<Op>> put; - metrics::LoadMetric<OpWithRequestSize<OpWithNotFound>> get; - metrics::LoadMetric<OpWithRequestSize<OpWithNotFound>> remove; + metrics::LoadMetric<PutMetricType> put; + metrics::LoadMetric<GetMetricType> get; + metrics::LoadMetric<RemoveMetricType> remove; metrics::LoadMetric<Op> removeLocation; metrics::LoadMetric<Op> statBucket; metrics::LoadMetric<Update> update; diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp index 3db3ec5dfcd..60b2c625d43 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp @@ -73,7 +73,7 @@ void ModifiedBucketChecker::configure( std::unique_ptr<vespa::config::content::core::StorServerConfig> newConfig) { - vespalib::LockGuard lock(_stateLock); + std::lock_guard lock(_stateLock); if (newConfig->bucketRecheckingChunkSize < 1) { throw config::InvalidConfigException( "Cannot have bucket rechecking chunk size of less than 1"); @@ -135,7 +135,7 @@ ModifiedBucketChecker::onInternalReply( const std::shared_ptr<api::InternalReply>& r) { if (r->getType() == RecheckBucketInfoReply::ID) { - vespalib::LockGuard guard(_stateLock); + std::lock_guard guard(_stateLock); assert(_pendingRequests > 0); --_pendingRequests; if (_pendingRequests == 0 && moreChunksRemaining()) { @@ -158,7 +158,7 @@ ModifiedBucketChecker::requestModifiedBucketsFromProvider(document::BucketSpace return false; } { - vespalib::LockGuard guard(_stateLock); + std::lock_guard guard(_stateLock); _rechecksNotStarted.reset(bucketSpace, result.getList()); } return true; @@ -202,7 +202,7 @@ ModifiedBucketChecker::tick() // we want getModifiedBuckets() to called outside the lock. bool shouldRequestFromProvider = false; { - vespalib::LockGuard guard(_stateLock); + std::lock_guard guard(_stateLock); if (!currentChunkFinished()) { return true; } @@ -216,7 +216,7 @@ ModifiedBucketChecker::tick() std::vector<RecheckBucketInfoCommand::SP> commandsToSend; { - vespalib::LockGuard guard(_stateLock); + std::lock_guard guard(_stateLock); if (moreChunksRemaining()) { nextRecheckChunk(commandsToSend); } diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h index 05389cec3da..7114d1f676f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h @@ -86,7 +86,7 @@ private: framework::Thread::UP _thread; config::ConfigFetcher _configFetcher; vespalib::Monitor _monitor; - vespalib::Lock _stateLock; + std::mutex _stateLock; CyclicBucketSpaceIterator::UP _bucketSpaces; BucketIdListResult _rechecksNotStarted; size_t _pendingRequests; diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index a39abc7fa58..6cf0ec194e1 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -177,6 +177,9 @@ PersistenceThread::handlePut(api::PutCommand& cmd, MessageTracker::UP trackerUP) metrics.request_size.addValue(cmd.getApproxByteSize()); if (tasConditionExists(cmd) && !tasConditionMatches(cmd, tracker, tracker.context())) { + // Will also count condition parse failures etc as TaS failures, but + // those results _will_ increase the error metrics as well. + metrics.test_and_set_failed.inc(); return trackerUP; } @@ -204,6 +207,7 @@ PersistenceThread::handleRemove(api::RemoveCommand& cmd, MessageTracker::UP trac metrics.request_size.addValue(cmd.getApproxByteSize()); if (tasConditionExists(cmd) && !tasConditionMatches(cmd, tracker, tracker.context())) { + metrics.test_and_set_failed.inc(); return trackerUP; } @@ -243,6 +247,7 @@ PersistenceThread::handleUpdate(api::UpdateCommand& cmd, MessageTracker::UP trac metrics.request_size.addValue(cmd.getApproxByteSize()); if (tasConditionExists(cmd) && !tasConditionMatches(cmd, tracker, tracker.context(), cmd.getUpdate()->getCreateIfNonExistent())) { + metrics.test_and_set_failed.inc(); return trackerUP; } diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 63ac5405fab..775e05488a9 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -68,12 +68,22 @@ MessageTracker::setMetric(FileStorThreadMetrics::Op& metric) { MessageTracker::~MessageTracker() = default; +bool MessageTracker::count_result_as_failure() const noexcept { + // Explicitly don't treat TaS failures as regular failures. These are tracked separately + // for operations that support TaS conditions. + if (hasReply() && getReply().getResult().failed()) { + return (getReply().getResult().getResult() != api::ReturnCode::TEST_AND_SET_CONDITION_FAILED); + } + return (getResult().failed() + && (getResult().getResult() != api::ReturnCode::TEST_AND_SET_CONDITION_FAILED)); +} + void MessageTracker::sendReply() { if ( ! _msg->getType().isReply()) { generateReply(static_cast<api::StorageCommand &>(*_msg)); } - if ((hasReply() && getReply().getResult().failed()) || getResult().failed()) { + if (count_result_as_failure()) { _env._metrics.failedOperations.inc(); } vespalib::duration duration = vespalib::from_s(_timer.getElapsedTimeAsDouble()/1000.0); @@ -140,7 +150,11 @@ MessageTracker::generateReply(api::StorageCommand& cmd) } if (!_reply->getResult().success()) { - _metric->failed.inc(); + // TaS failures are tracked separately and explicitly in the put/update/remove paths, + // so don't double-count them here. + if (_reply->getResult().getResult() != api::ReturnCode::TEST_AND_SET_CONDITION_FAILED) { + _metric->failed.inc(); + } LOGBP(debug, "Failed to handle command %s: %s", cmd.toString().c_str(), _result.toString().c_str()); diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index a57ef186b46..51eb2b4b590 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -78,6 +78,9 @@ public: private: MessageTracker(PersistenceUtil & env, MessageSender & replySender, bool updateBucketInfo, FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg); + + [[nodiscard]] bool count_result_as_failure() const noexcept; + bool _sendReply; bool _updateBucketInfo; FileStorHandler::BucketLockInterface::SP _bucketLock; diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index bc9708f70a2..8a0cffedd05 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -72,7 +72,7 @@ Bouncer::configure(std::unique_ptr<vespa::config::content::core::StorBouncerConf { log_config_received(*config); validateConfig(*config); - vespalib::LockGuard lock(_lock); + std::lock_guard lock(_lock); _config = std::move(config); } @@ -248,7 +248,7 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) bool abortLoadWhenClusterDown; int feedPriorityLowerBound; { - vespalib::LockGuard lock(_lock); + std::lock_guard lock(_lock); state = &getDerivedNodeState(msg->getBucket().getBucketSpace()).getState(); maxClockSkewInSeconds = _config->maxClockSkewSeconds; abortLoadWhenClusterDown = _config->stopExternalLoadWhenClusterDown; @@ -325,7 +325,7 @@ deriveNodeState(const lib::NodeState &reportedNodeState, void Bouncer::handleNewState() { - vespalib::LockGuard lock(_lock); + std::lock_guard lock(_lock); const auto reportedNodeState = *_component.getStateUpdater().getReportedNodeState(); const auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); const auto &clusterState = *clusterStateBundle->getBaselineClusterState(); diff --git a/storage/src/vespa/storage/storageserver/bouncer.h b/storage/src/vespa/storage/storageserver/bouncer.h index 19c8e084c7d..ee51114fcbe 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.h +++ b/storage/src/vespa/storage/storageserver/bouncer.h @@ -17,7 +17,6 @@ #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storage/config/config-stor-bouncer.h> -#include <vespa/vespalib/util/sync.h> #include <unordered_map> namespace config { class ConfigUri; } @@ -32,8 +31,8 @@ class Bouncer : public StorageLink, { std::unique_ptr<vespa::config::content::core::StorBouncerConfig> _config; StorageComponent _component; - vespalib::Lock _lock; - lib::NodeState _baselineNodeState; + std::mutex _lock; + lib::NodeState _baselineNodeState; using BucketSpaceNodeStateMapping = std::unordered_map<document::BucketSpace, lib::NodeState, document::BucketSpace::hash>; BucketSpaceNodeStateMapping _derivedNodeStates; const lib::State* _clusterState; diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp index 66e0209d4b7..aec0adf790c 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp @@ -57,7 +57,7 @@ ChangedBucketOwnershipHandler::configure( void ChangedBucketOwnershipHandler::reloadClusterState() { - vespalib::LockGuard guard(_stateLock); + std::lock_guard guard(_stateLock); const auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); setCurrentOwnershipWithStateNoLock(*clusterStateBundle); } @@ -258,7 +258,7 @@ ChangedBucketOwnershipHandler::onSetSystemState( // can get through in the off-case that the lower level storage links // don't apply the state immediately for some reason. { - vespalib::LockGuard guard(_stateLock); + std::lock_guard guard(_stateLock); oldOwnership = _currentOwnership; setCurrentOwnershipWithStateNoLock(stateCmd->getClusterStateBundle()); newOwnership = _currentOwnership; @@ -301,7 +301,7 @@ ChangedBucketOwnershipHandler::onSetSystemState( void ChangedBucketOwnershipHandler::storageDistributionChanged() { - vespalib::LockGuard guard(_stateLock); + std::lock_guard guard(_stateLock); _currentOwnership = std::make_shared<OwnershipState>( _component.getBucketSpaceRepo(), _currentState); } @@ -345,7 +345,7 @@ ChangedBucketOwnershipHandler::isMutatingExternalOperation( ChangedBucketOwnershipHandler::OwnershipState::CSP ChangedBucketOwnershipHandler::getCurrentOwnershipState() const { - vespalib::LockGuard guard(_stateLock); + std::lock_guard guard(_stateLock); return _currentOwnership; } diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h index 1e524b6b2fc..0fc456b515c 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h @@ -4,7 +4,6 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/storage/common/storagelink.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/metrics/metrics.h> #include <vespa/config/config.h> #include <vespa/config-persistence.h> @@ -110,9 +109,9 @@ public: private: ServiceLayerComponent _component; - Metrics _metrics; + Metrics _metrics; config::ConfigFetcher _configFetcher; - vespalib::Lock _stateLock; + mutable std::mutex _stateLock; std::shared_ptr<const lib::ClusterStateBundle> _currentState; OwnershipState::CSP _currentOwnership; diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index cfd74bacf16..8a7cc8ffc61 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -187,7 +187,7 @@ CommunicationManager::handleReply(std::unique_ptr<mbus::Reply> reply) if (protocolName == documentapi::DocumentProtocol::NAME) { std::shared_ptr<api::StorageCommand> originalCommand; { - vespalib::LockGuard lock(_messageBusSentLock); + std::lock_guard lock(_messageBusSentLock); typedef std::map<api::StorageMessage::Id, api::StorageCommand::SP> MessageMap; MessageMap::iterator iter(_messageBusSent.find(reply->getContext().value.UINT64)); if (iter != _messageBusSent.end()) { @@ -601,7 +601,7 @@ CommunicationManager::sendCommand( mbusMsg->setRetryEnabled(false); { - vespalib::LockGuard lock(_messageBusSentLock); + std::lock_guard lock(_messageBusSentLock); _messageBusSent[msg->getMsgId()] = msg; } sendMessageBusMessage(msg, std::move(mbusMsg), address.getRoute()); diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 7ac9d575ee6..b10b12f404e 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -112,7 +112,7 @@ private: std::unique_ptr<mbus::DestinationSession> _messageBusSession; std::unique_ptr<mbus::SourceSession> _sourceSession; - vespalib::Lock _messageBusSentLock; + std::mutex _messageBusSentLock; std::map<api::StorageMessage::Id, std::shared_ptr<api::StorageCommand> > _messageBusSent; config::ConfigUri _configUri; @@ -137,7 +137,7 @@ public: void dispatch_sync(std::shared_ptr<api::StorageMessage> msg) override; void dispatch_async(std::shared_ptr<api::StorageMessage> msg) override; - mbus::RPCMessageBus& getMessageBus() { assert(_mbus.get()); return *_mbus; } + mbus::RPCMessageBus& getMessageBus() { return *_mbus; } const PriorityConverter& getPriorityConverter() const { return _docApiConverter.getPriorityConverter(); } /** diff --git a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h index 0800cc9612e..34e30aaea48 100644 --- a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h +++ b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h @@ -21,7 +21,7 @@ public: using BucketSpaceMapping = std::unordered_map<vespalib::string, document::BucketSpace, vespalib::hash<vespalib::string>>; const BucketSpaceMapping _type_to_space; public: - explicit ConfigurableBucketResolver(BucketSpaceMapping type_to_space) + explicit ConfigurableBucketResolver(BucketSpaceMapping type_to_space) noexcept : _type_to_space(std::move(type_to_space)) {} diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 83e31b78954..7999514ac00 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> #include <algorithm> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".mergethrottler"); @@ -214,7 +215,7 @@ MergeThrottler::MergeThrottler( void MergeThrottler::configure(std::unique_ptr<vespa::config::content::core::StorServerConfig> newConfig) { - vespalib::LockGuard lock(_stateLock); + std::lock_guard lock(_stateLock); if (newConfig->maxMergesPerNode < 1) { throw config::InvalidConfigException("Cannot have a max merge count of less than 1"); @@ -277,7 +278,7 @@ MergeThrottler::onClose() _closing = true; } if (LOG_WOULD_LOG(debug)) { - vespalib::LockGuard lock(_stateLock); + std::lock_guard lock(_stateLock); LOG(debug, "onClose; active: %zu, queued: %zu", _merges.size(), _queue.size()); } @@ -700,7 +701,7 @@ void MergeThrottler::backpressure_bounce_all_queued_merges(MessageGuard& guard) } bool MergeThrottler::backpressure_mode_active() const { - vespalib::LockGuard lock(_stateLock); + std::lock_guard lock(_stateLock); return backpressure_mode_active_no_lock(); } @@ -1072,7 +1073,7 @@ MergeThrottler::onDown(const std::shared_ptr<api::StorageMessage>& msg) lock.broadcast(); return true; } else if (isDiffCommand(*msg)) { - vespalib::LockGuard lock(_stateLock); + std::lock_guard lock(_stateLock); auto& cmd = static_cast<api::StorageCommand&>(*msg); if (bucketIsUnknownOrAborted(cmd.getBucket())) { sendUp(makeAbortReply(cmd, "no state recorded for bucket in merge " @@ -1264,7 +1265,7 @@ void MergeThrottler::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath&) const { - vespalib::LockGuard lock(_stateLock); + std::lock_guard lock(_stateLock); { out << "<p>Max pending: " << _throttlePolicy->getMaxPendingCount() diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index d62e9a042b2..b2087600105 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -155,7 +155,7 @@ private: mbus::StaticThrottlePolicy::UP _throttlePolicy; uint64_t _queueSequence; // TODO: move into a stable priority queue class vespalib::Monitor _messageLock; - vespalib::Lock _stateLock; + mutable std::mutex _stateLock; config::ConfigFetcher _configFetcher; // Messages pending to be processed by the worker thread std::vector<api::StorageMessage::SP> _messagesDown; @@ -205,7 +205,7 @@ public: mbus::StaticThrottlePolicy& getThrottlePolicy() { return *_throttlePolicy; } // For unit testing only vespalib::Monitor& getMonitor() { return _messageLock; } - vespalib::Lock& getStateLock() { return _stateLock; } + std::mutex & getStateLock() { return _stateLock; } Metrics& getMetrics() { return *_metrics; } std::size_t getMaxQueueSize() const { return _maxQueueSize; } diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp index b6bceabf7a1..3e1fc34d470 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.cpp +++ b/storage/src/vespa/storage/storageserver/opslogger.cpp @@ -42,7 +42,7 @@ OpsLogger::onClose() void OpsLogger::configure(std::unique_ptr<vespa::config::content::core::StorOpsloggerConfig> config) { - vespalib::LockGuard lock(_lock); + std::lock_guard lock(_lock); // If no change in state, ignore if (config->targetfile == _fileName) return; // If a change we need to close old handle if open @@ -79,7 +79,7 @@ OpsLogger::onPutReply(const std::shared_ptr<api::PutReply>& msg) << "\tPUT\t" << msg->getDocumentId() << "\t" << msg->getResult() << "\n"; { - vespalib::LockGuard lock(_lock); + std::lock_guard lock(_lock); if (_targetFile == nullptr) return false; fwrite(ost.str().c_str(), ost.str().length(), 1, _targetFile); fflush(_targetFile); @@ -96,7 +96,7 @@ OpsLogger::onUpdateReply(const std::shared_ptr<api::UpdateReply>& msg) << "\tUPDATE\t" << msg->getDocumentId() << "\t" << msg->getResult() << "\n"; { - vespalib::LockGuard lock(_lock); + std::lock_guard lock(_lock); if (_targetFile == nullptr) return false; fwrite(ost.str().c_str(), ost.str().length(), 1, _targetFile); fflush(_targetFile); @@ -113,7 +113,7 @@ OpsLogger::onRemoveReply(const std::shared_ptr<api::RemoveReply>& msg) << "\tREMOVE\t" << msg->getDocumentId() << "\t" << msg->getResult() << "\n"; { - vespalib::LockGuard lock(_lock); + std::lock_guard lock(_lock); if (_targetFile == nullptr) return false; fwrite(ost.str().c_str(), ost.str().length(), 1, _targetFile); fflush(_targetFile); @@ -130,7 +130,7 @@ OpsLogger::onGetReply(const std::shared_ptr<api::GetReply>& msg) << "\tGET\t" << msg->getDocumentId() << "\t" << msg->getResult() << "\n"; { - vespalib::LockGuard lock(_lock); + std::lock_guard lock(_lock); if (_targetFile == nullptr) return false; fwrite(ost.str().c_str(), ost.str().length(), 1, _targetFile); fflush(_targetFile); diff --git a/storage/src/vespa/storage/storageserver/opslogger.h b/storage/src/vespa/storage/storageserver/opslogger.h index 1e903bb1ef3..cecadf9ed1d 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.h +++ b/storage/src/vespa/storage/storageserver/opslogger.h @@ -13,7 +13,6 @@ #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageapi/message/state.h> #include <vespa/storage/config/config-stor-opslogger.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/config/config.h> namespace storage { @@ -23,7 +22,7 @@ class OpsLogger : public StorageLink, public: explicit OpsLogger(StorageComponentRegister&, const config::ConfigUri & configUri); - ~OpsLogger(); + ~OpsLogger() override; void onClose() override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; @@ -36,9 +35,9 @@ public: bool onDown(const std::shared_ptr<api::StorageMessage>&) override { return false; }; void configure(std::unique_ptr<vespa::config::content::core::StorOpsloggerConfig> config) override; private: - vespalib::Lock _lock; - std::string _fileName; - FILE* _targetFile; + std::mutex _lock; + std::string _fileName; + FILE * _targetFile; framework::Component _component; config::ConfigFetcher _configFetcher; diff --git a/storage/src/vespa/storage/storageserver/priorityconverter.cpp b/storage/src/vespa/storage/storageserver/priorityconverter.cpp index 8d0f18b4157..41bea1b44df 100644 --- a/storage/src/vespa/storage/storageserver/priorityconverter.cpp +++ b/storage/src/vespa/storage/storageserver/priorityconverter.cpp @@ -31,7 +31,7 @@ PriorityConverter::toStoragePriority(documentapi::Priority::Value documentApiPri documentapi::Priority::Value PriorityConverter::toDocumentPriority(uint8_t storagePriority) const { - vespalib::LockGuard guard(_mutex); + std::lock_guard guard(_mutex); std::map<uint8_t, documentapi::Priority::Value>::const_iterator iter = _reverseMapping.lower_bound(storagePriority); @@ -63,7 +63,7 @@ PriorityConverter::configure(std::unique_ptr<vespa::config::content::core::StorP _mapping[documentapi::Priority::PRI_VERY_LOW] = config->veryLow; _mapping[documentapi::Priority::PRI_LOWEST] = config->lowest; - vespalib::LockGuard guard(_mutex); + std::lock_guard guard(_mutex); _reverseMapping.clear(); _reverseMapping[config->highest] = documentapi::Priority::PRI_HIGHEST; _reverseMapping[config->veryHigh] = documentapi::Priority::PRI_VERY_HIGH; diff --git a/storage/src/vespa/storage/storageserver/priorityconverter.h b/storage/src/vespa/storage/storageserver/priorityconverter.h index f4cfb7488e2..08669c672d5 100644 --- a/storage/src/vespa/storage/storageserver/priorityconverter.h +++ b/storage/src/vespa/storage/storageserver/priorityconverter.h @@ -5,9 +5,9 @@ #include <vespa/storage/config/config-stor-prioritymapping.h> #include <vespa/config/helper/configfetcher.h> #include <vespa/documentapi/messagebus/priority.h> -#include <vespa/vespalib/util/sync.h> #include <atomic> #include <array> +#include <mutex> namespace config {class ConfigUri; } @@ -20,8 +20,8 @@ class PriorityConverter public: typedef vespa::config::content::core::StorPrioritymappingConfig Config; - PriorityConverter(const config::ConfigUri& configUri); - ~PriorityConverter(); + explicit PriorityConverter(const config::ConfigUri& configUri); + ~PriorityConverter() override; /** Converts the given priority into a storage api priority number. */ uint8_t toStoragePriority(documentapi::Priority::Value) const; @@ -40,7 +40,7 @@ private: std::array<std::atomic<uint8_t>, PRI_ENUM_SIZE> _mapping; std::map<uint8_t, documentapi::Priority::Value> _reverseMapping; - vespalib::Lock _mutex; + mutable std::mutex _mutex; config::ConfigFetcher _configFetcher; }; diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp index 8896df7a155..553132507ab 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp @@ -66,8 +66,8 @@ SharedRpcResources::SharedRpcResources(const config::ConfigUri& config_uri, : _thread_pool(std::make_unique<FastOS_ThreadPool>(1024*60)), _transport(std::make_unique<FNET_Transport>(rpc_thread_pool_size)), _orb(std::make_unique<FRT_Supervisor>(_transport.get())), - _slobrok_register(std::make_unique<slobrok::api::RegisterAPI>(*_orb, config_uri)), - _slobrok_mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, config_uri)), + _slobrok_register(std::make_unique<slobrok::api::RegisterAPI>(*_orb, slobrok::ConfiguratorFactory(config_uri))), + _slobrok_mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, slobrok::ConfiguratorFactory(config_uri))), _target_factory(std::make_unique<RpcTargetFactoryImpl>(*_orb)), _hostname(vespalib::HostName::get()), _rpc_server_port(rpc_server_port), diff --git a/storage/src/vespa/storage/storageserver/service_layer_error_listener.h b/storage/src/vespa/storage/storageserver/service_layer_error_listener.h index 6995459e333..5dd01329ff4 100644 --- a/storage/src/vespa/storage/storageserver/service_layer_error_listener.h +++ b/storage/src/vespa/storage/storageserver/service_layer_error_listener.h @@ -23,7 +23,7 @@ class ServiceLayerErrorListener : public ProviderErrorListener { std::atomic<bool> _shutdown_initiated; public: ServiceLayerErrorListener(StorageComponent& component, - MergeThrottler& merge_throttler) + MergeThrottler& merge_throttler) noexcept : _component(component), _merge_throttler(merge_throttler), _shutdown_initiated(false) diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index 4f1db0e1b30..1edfcde393b 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -91,7 +91,7 @@ ServiceLayerNode::subscribeToConfigs() StorageNode::subscribeToConfigs(); _configFetcher.reset(new config::ConfigFetcher(_configUri.getContext())); - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); // Verify and set disk count if (_serverConfig->diskCount != 0 && _serverConfig->diskCount != _partitions.size()) diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index da221e312e2..db4dfd0323e 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -204,14 +204,14 @@ StateManager::getClusterStateBundle() const void StateManager::addStateListener(StateListener& listener) { - vespalib::LockGuard lock(_listenerLock); + std::lock_guard lock(_listenerLock); _stateListeners.push_back(&listener); } void StateManager::removeStateListener(StateListener& listener) { - vespalib::LockGuard lock(_listenerLock); + std::lock_guard lock(_listenerLock); for (auto it = _stateListeners.begin(); it != _stateListeners.end();) { if (*it == &listener) { it = _stateListeners.erase(it); @@ -224,7 +224,7 @@ StateManager::removeStateListener(StateListener& listener) struct StateManager::ExternalStateLock : public NodeStateUpdater::Lock { StateManager& _manager; - explicit ExternalStateLock(StateManager& manager) : _manager(manager) {} + explicit ExternalStateLock(StateManager& manager) noexcept : _manager(manager) {} ~ExternalStateLock() override { { vespalib::MonitorGuard lock(_manager._stateLock); @@ -282,7 +282,7 @@ StateManager::notifyStateListeners() if (_notifyingListeners) { return; } - vespalib::LockGuard listenerLock(_listenerLock); + std::lock_guard listenerLock(_listenerLock); _notifyingListeners = true; lib::NodeState::SP newState; while (true) { diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index 3d231c070ef..b6f43eb270e 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -45,7 +45,7 @@ class StateManager : public NodeStateUpdater, StorageComponent _component; metrics::MetricManager& _metricManager; vespalib::Monitor _stateLock; - vespalib::Lock _listenerLock; + std::mutex _listenerLock; std::shared_ptr<lib::NodeState> _nodeState; std::shared_ptr<lib::NodeState> _nextNodeState; using ClusterStateBundle = lib::ClusterStateBundle; diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index aa50391c037..c1cfc97bab6 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -129,7 +129,7 @@ StorageNode::subscribeToConfigs() _configFetcher->start(); - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); _serverConfig = std::move(_newServerConfig); _clusterConfig = std::move(_newClusterConfig); _distributionConfig = std::move(_newDistributionConfig); @@ -252,7 +252,7 @@ StorageNode::initializeStatusWebServer() void StorageNode::setNewDocumentRepo(const std::shared_ptr<const document::DocumentTypeRepo>& repo) { - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); _context.getComponentRegister().setDocumentTypeRepo(repo); if (_communicationManager != nullptr) { _communicationManager->updateMessagebusProtocol(repo); @@ -280,7 +280,7 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) { // Make sure we don't conflict with initialize or shutdown threads. (void) initGuard; - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); assert(_chain); // If we get here, initialize is done running. We have to handle changes @@ -476,7 +476,7 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { // to a variable where we can find it later when processing config // updates { - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); _newServerConfig = std::move(config); } if (_serverConfig) { @@ -488,7 +488,7 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { log_config_received(*config); { - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); _newClusterConfig = std::move(config); } if (_clusterConfig) { @@ -500,7 +500,7 @@ void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { log_config_received(*config); { - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); _newDistributionConfig = std::move(config); } if (_distributionConfig) { @@ -512,7 +512,7 @@ void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { void StorageNode::configure(std::unique_ptr<StorPrioritymappingConfig> config) { log_config_received(*config); { - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); _newPriorityConfig = std::move(config); } if (_priorityConfig) { @@ -530,7 +530,7 @@ StorageNode::configure(std::unique_ptr<document::DocumenttypesConfig> config, if (!hasChanged) return; { - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); _newDoctypesConfig = std::move(config); } if (_doctypesConfig) { @@ -542,7 +542,7 @@ StorageNode::configure(std::unique_ptr<document::DocumenttypesConfig> config, void StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { log_config_received(*config); { - vespalib::LockGuard configLockGuard(_configLock); + std::lock_guard configLockGuard(_configLock); _newBucketSpacesConfig = std::move(config); } if (_bucketSpacesConfig) { diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 91a2bae3190..adc55266588 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -144,7 +144,7 @@ private: protected: // Lock taken while doing configuration of the server. - vespalib::Lock _configLock; + std::mutex _configLock; std::mutex _initial_config_mutex; using InitialGuard = std::lock_guard<std::mutex>; // Current running config. Kept, such that we can see what has been diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index 80e74e890a1..1ac0248961a 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -34,7 +34,7 @@ RecoveryVisitor::handleDocuments(const document::BucketId& bid, std::vector<spi::DocEntry::UP>& entries, HitCounter& hitCounter) { - vespalib::LockGuard guard(_mutex); + std::lock_guard guard(_mutex); LOG(debug, "Visitor %s handling block of %zu documents.", _id.c_str(), entries.size()); @@ -87,7 +87,7 @@ void RecoveryVisitor::completedBucket(const document::BucketId& bid, HitCounter& LOG(debug, "Finished bucket %s", bid.toString().c_str()); { - vespalib::LockGuard guard(_mutex); + std::lock_guard guard(_mutex); CommandMap::iterator iter = _activeCommands.find(bid); diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.h b/storage/src/vespa/storage/visiting/recoveryvisitor.h index 1da2acfed9c..fd296c30361 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.h +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.h @@ -33,7 +33,7 @@ private: using CommandMap = std::map<document::BucketId, CommandPtr>; CommandMap _activeCommands; - vespalib::Lock _mutex; + std::mutex _mutex; }; struct RecoveryVisitorFactory : public VisitorFactory { diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index dcd8cc4ba39..982bcd78f6f 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -13,6 +13,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <unordered_map> #include <sstream> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.visitor"); diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index 1a1f1498578..7a450508375 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -10,6 +10,7 @@ #include <vespa/config/common/exceptions.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".visitor.manager"); @@ -660,7 +661,7 @@ VisitorManager::reportHtmlStatus(std::ostream& out, } // Only one can access status at a time as _statusRequest only holds // answers from one request at a time - vespalib::LockGuard sync(_statusLock); + std::lock_guard sync(_statusLock); vespalib::MonitorGuard waiter(_statusMonitor); // Send all subrequests uint32_t parts = _visitorThread.size(); diff --git a/storage/src/vespa/storage/visiting/visitormanager.h b/storage/src/vespa/storage/visiting/visitormanager.h index 3675a824e1d..70ba95396fc 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.h +++ b/storage/src/vespa/storage/visiting/visitormanager.h @@ -75,10 +75,9 @@ private: framework::MicroSecTime> > _recentlyDeletedVisitors; framework::MicroSecTime _recentlyDeletedMaxTime; - mutable vespalib::Lock _statusLock; // Only one can get status at a time + mutable std::mutex _statusLock; // Only one can get status at a time mutable vespalib::Monitor _statusMonitor; // Notify when done - mutable std::vector<std::shared_ptr<RequestStatusPageReply> > - _statusRequest; + mutable std::vector<std::shared_ptr<RequestStatusPageReply> > _statusRequest; bool _enforceQueueUse; VisitorFactory::Map _visitorFactories; diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp index c3980784106..7466d5c603e 100644 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp +++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp @@ -5,7 +5,7 @@ namespace storage::api { -BucketInfo::BucketInfo() +BucketInfo::BucketInfo() noexcept : _lastModified(0), _checksum(0), _docCount(0), @@ -17,7 +17,7 @@ BucketInfo::BucketInfo() {} BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, - uint32_t totDocSize) + uint32_t totDocSize) noexcept : _lastModified(0), _checksum(checksum), _docCount(docCount), @@ -30,7 +30,7 @@ BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, uint32_t metaCount, - uint32_t usedFileSize) + uint32_t usedFileSize) noexcept : _lastModified(0), _checksum(checksum), _docCount(docCount), @@ -44,7 +44,7 @@ BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, uint32_t metaCount, uint32_t usedFileSize, - bool ready, bool active) + bool ready, bool active) noexcept : _lastModified(0), _checksum(checksum), _docCount(docCount), @@ -58,7 +58,7 @@ BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, uint32_t metaCount, uint32_t usedFileSize, - bool ready, bool active, Timestamp lastModified) + bool ready, bool active, Timestamp lastModified) noexcept : _lastModified(lastModified), _checksum(checksum), _docCount(docCount), @@ -69,9 +69,9 @@ BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, _active(active) {} -BucketInfo::BucketInfo(const BucketInfo &) = default; -BucketInfo & BucketInfo::operator = (const BucketInfo &) = default; -BucketInfo::~BucketInfo() {} +BucketInfo::BucketInfo(const BucketInfo &) noexcept = default; +BucketInfo & BucketInfo::operator = (const BucketInfo &) noexcept = default; +BucketInfo::~BucketInfo() = default; bool BucketInfo::operator==(const BucketInfo& info) const diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h index a80595527e2..f15d99588ca 100644 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h +++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h @@ -31,18 +31,18 @@ class BucketInfo : public vespalib::AsciiPrintable bool _active; public: - BucketInfo(); - BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize); + BucketInfo() noexcept; + BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize) noexcept; BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, - uint32_t metaCount, uint32_t usedFileSize); + uint32_t metaCount, uint32_t usedFileSize) noexcept; BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, uint32_t metaCount, uint32_t usedFileSize, - bool ready, bool active); + bool ready, bool active) noexcept; BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, uint32_t metaCount, uint32_t usedFileSize, - bool ready, bool active, Timestamp lastModified); - BucketInfo(const BucketInfo &); - BucketInfo & operator = (const BucketInfo &); + bool ready, bool active, Timestamp lastModified) noexcept; + BucketInfo(const BucketInfo &) noexcept; + BucketInfo & operator = (const BucketInfo &) noexcept; ~BucketInfo(); Timestamp getLastModified() const { return _lastModified; } diff --git a/storageapi/src/vespa/storageapi/defs.h b/storageapi/src/vespa/storageapi/defs.h index e5445f22870..531a1b13120 100644 --- a/storageapi/src/vespa/storageapi/defs.h +++ b/storageapi/src/vespa/storageapi/defs.h @@ -8,8 +8,7 @@ #include <cstdint> -namespace storage { -namespace api { +namespace storage:: api { typedef uint64_t Timestamp; typedef uint32_t VisitorId; @@ -17,4 +16,3 @@ typedef uint32_t VisitorId; const Timestamp MAX_TIMESTAMP = (Timestamp)-1ll; } -} diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index 45af2296e8a..2b09967d95b 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -104,10 +104,10 @@ public: uint16_t index; bool sourceOnly; - Node(uint16_t index_, bool sourceOnly_ = false) + Node(uint16_t index_, bool sourceOnly_ = false) noexcept : index(index_), sourceOnly(sourceOnly_) {} - bool operator==(const Node& n) const + bool operator==(const Node& n) const noexcept { return (index == n.index && sourceOnly == n.sourceOnly); } }; @@ -123,7 +123,7 @@ public: Timestamp maxTimestamp, uint32_t clusterStateVersion = 0, const std::vector<uint16_t>& chain = std::vector<uint16_t>()); - ~MergeBucketCommand(); + ~MergeBucketCommand() override; const std::vector<Node>& getNodes() const { return _nodes; } Timestamp getMaxTimestamp() const { return _maxTimestamp; } diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h index 7189cc67195..67c41a0cc4d 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.h +++ b/storageapi/src/vespa/storageapi/message/visitor.h @@ -186,12 +186,12 @@ public: document::BucketId bucketId; Timestamp timestamp; - BucketTimestampPair() : bucketId(), timestamp(0) {} - BucketTimestampPair(const document::BucketId& bucket, - const Timestamp& ts) - : bucketId(bucket), timestamp(ts) {} + BucketTimestampPair() noexcept : bucketId(), timestamp(0) {} + BucketTimestampPair(const document::BucketId& bucket, const Timestamp& ts) noexcept + : bucketId(bucket), timestamp(ts) + {} - bool operator==(const BucketTimestampPair& other) const { + bool operator==(const BucketTimestampPair& other) const noexcept { return (bucketId == other.bucketId && timestamp && other.timestamp); } }; @@ -203,7 +203,7 @@ private: public: VisitorInfoCommand(); - ~VisitorInfoCommand(); + ~VisitorInfoCommand() override; void setErrorCode(const ReturnCode& code) { _error = code; } void setCompleted() { _completed = true; } diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index 1f1a2c602de..0f5546a5510 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -4,7 +4,6 @@ #include <vespa/messagebus/routing/verbatimdirective.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_fun.h> #include <sstream> diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h b/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h index bf810d754d6..24132f710fd 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h @@ -8,11 +8,9 @@ #pragma once #include <vespa/storageframework/generic/clock/clock.h> -#include <vespa/vespalib/util/sync.h> +#include <mutex> -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { struct FakeClock : public framework::Clock { enum Mode { @@ -25,45 +23,45 @@ struct FakeClock : public framework::Clock { private: Mode _mode; framework::MicroSecTime _absoluteTime; - mutable time_t _cycleCount; - vespalib::Lock _lock; + mutable time_t _cycleCount; + mutable std::mutex _lock; public: FakeClock(Mode m = FAKE_ABSOLUTE, framework::MicroSecTime startTime = framework::MicroSecTime(1)); void setMode(Mode m) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _mode = m; } virtual void setFakeCycleMode() { setMode(FAKE_ABSOLUTE_CYCLE); } virtual void setAbsoluteTimeInSeconds(uint32_t seconds) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _absoluteTime = framework::MicroSecTime(seconds * uint64_t(1000000)); _cycleCount = 0; _mode = FAKE_ABSOLUTE; } virtual void setAbsoluteTimeInMicroSeconds(uint64_t usecs) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _absoluteTime = framework::MicroSecTime(usecs); _cycleCount = 0; _mode = FAKE_ABSOLUTE; } virtual void addMilliSecondsToTime(uint64_t ms) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _absoluteTime += framework::MicroSecTime(ms * 1000); } virtual void addSecondsToTime(uint32_t nr) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _absoluteTime += framework::MicroSecTime(nr * uint64_t(1000000)); } framework::MicroSecTime getTimeInMicros() const override { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (_mode == FAKE_ABSOLUTE) return _absoluteTime; return _absoluteTime + framework::MicroSecTime(1000000 * _cycleCount++); } @@ -80,7 +78,5 @@ public: } }; -} // defaultimplementation -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp index 2595514e6f4..e4560370a01 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp @@ -4,6 +4,7 @@ #include <vespa/storageframework/storageframework.h> #include <vespa/metrics/metricmanager.h> #include <vespa/vespalib/util/exceptions.h> +#include <cassert> namespace storage::framework::defaultimplementation { @@ -24,7 +25,7 @@ ComponentRegisterImpl::~ComponentRegisterImpl() = default; void ComponentRegisterImpl::registerComponent(ManagedComponent& mc) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _components.push_back(&mc); if (_clock) { mc.setClock(*_clock); @@ -41,7 +42,7 @@ ComponentRegisterImpl::registerComponent(ManagedComponent& mc) void ComponentRegisterImpl::requestShutdown(vespalib::stringref reason) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); if (_shutdownListener) { _shutdownListener->requestShutdown(reason); } @@ -52,7 +53,7 @@ ComponentRegisterImpl::setMetricManager(metrics::MetricManager& mm) { std::vector<ManagedComponent*> components; { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); assert(_metricManager == nullptr); components = _components; _metricManager = &mm; @@ -69,7 +70,7 @@ ComponentRegisterImpl::setMetricManager(metrics::MetricManager& mm) void ComponentRegisterImpl::setClock(Clock& c) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); assert(_clock == nullptr); _clock = &c; for (auto* component : _components) { @@ -80,7 +81,7 @@ ComponentRegisterImpl::setClock(Clock& c) void ComponentRegisterImpl::setThreadPool(ThreadPool& tp) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); assert(_threadPool == nullptr); _threadPool = &tp; for (auto* component : _components) { @@ -91,7 +92,7 @@ ComponentRegisterImpl::setThreadPool(ThreadPool& tp) void ComponentRegisterImpl::setUpgradeFlag(UpgradeFlags flag) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); _upgradeFlag = flag; for (auto* component : _components) { component->setUpgradeFlag(_upgradeFlag); @@ -101,7 +102,7 @@ ComponentRegisterImpl::setUpgradeFlag(UpgradeFlags flag) const StatusReporter* ComponentRegisterImpl::getStatusReporter(vespalib::stringref id) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); for (auto* component : _components) { if ((component->getStatusReporter() != nullptr) && (component->getStatusReporter()->getId() == id)) @@ -116,7 +117,7 @@ std::vector<const StatusReporter*> ComponentRegisterImpl::getStatusReporters() { std::vector<const StatusReporter*> reporters; - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); for (auto* component : _components) { if (component->getStatusReporter() != nullptr) { reporters.emplace_back(component->getStatusReporter()); @@ -152,7 +153,7 @@ ComponentRegisterImpl::registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, SecondTime period) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); auto hookPtr = std::make_unique<MetricHookWrapper>(name, hook); _metricManager->addMetricUpdateHook(*hookPtr, period.getTime()); _hooks.emplace_back(std::move(hookPtr)); @@ -167,7 +168,7 @@ ComponentRegisterImpl::getMetricManagerLock() void ComponentRegisterImpl::registerShutdownListener(ShutdownListener& listener) { - vespalib::LockGuard lock(_componentLock); + std::lock_guard lock(_componentLock); assert(_shutdownListener == nullptr); _shutdownListener = &listener; } diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h index b5f79313dfb..3adef5f4838 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h @@ -21,7 +21,6 @@ #include <vespa/storageframework/generic/component/managedcomponent.h> #include <vespa/storageframework/generic/metric/metricregistrator.h> #include <vespa/storageframework/generic/status/statusreportermap.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/metrics/metricset.h> namespace metrics { @@ -43,7 +42,7 @@ class ComponentRegisterImpl : public virtual ComponentRegister, public StatusReporterMap, public MetricRegistrator { - vespalib::Lock _componentLock; + std::mutex _componentLock; std::vector<ManagedComponent*> _components; metrics::MetricSet _topMetricSet; diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp index 2e90e1ae3ee..5c8e70f2773 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "testcomponentregister.h" +#include <cassert> namespace storage::framework::defaultimplementation { diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp index affeae44c04..e4a1188030c 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp @@ -21,7 +21,7 @@ ThreadPoolImpl::ThreadPoolImpl(Clock& clock) ThreadPoolImpl::~ThreadPoolImpl() { { - vespalib::LockGuard lock(_threadVectorLock); + std::lock_guard lock(_threadVectorLock); _stopping = true; for (ThreadImpl * thread : _threads) { thread->interrupt(); @@ -32,7 +32,7 @@ ThreadPoolImpl::~ThreadPoolImpl() } for (uint32_t i=0; true; i+=10) { { - vespalib::LockGuard lock(_threadVectorLock); + std::lock_guard lock(_threadVectorLock); if (_threads.empty()) break; } if (i > 1000) { @@ -49,7 +49,7 @@ Thread::UP ThreadPoolImpl::startThread(Runnable& runnable, vespalib::stringref id, uint64_t waitTimeMs, uint64_t maxProcessTime, int ticksBeforeWait) { - vespalib::LockGuard lock(_threadVectorLock); + std::lock_guard lock(_threadVectorLock); if (_stopping) { throw IllegalStateException("Threadpool is stopping", VESPA_STRLOC); } @@ -62,7 +62,7 @@ ThreadPoolImpl::startThread(Runnable& runnable, vespalib::stringref id, uint64_t void ThreadPoolImpl::visitThreads(ThreadVisitor& visitor) const { - vespalib::LockGuard lock(_threadVectorLock); + std::lock_guard lock(_threadVectorLock); for (const ThreadImpl * thread : _threads) { visitor.visitThread(thread->getId(), thread->getProperties(), thread->getTickData()); } @@ -71,7 +71,7 @@ ThreadPoolImpl::visitThreads(ThreadVisitor& visitor) const void ThreadPoolImpl::unregisterThread(ThreadImpl& t) { - vespalib::LockGuard lock(_threadVectorLock); + std::lock_guard lock(_threadVectorLock); std::vector<ThreadImpl*> threads; threads.reserve(_threads.size()); for (ThreadImpl * thread : _threads) { diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h index 4e973c2ad20..5f0ee523eff 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h @@ -4,7 +4,6 @@ #include <vespa/storageframework/generic/thread/threadpool.h> #include <vespa/fastos/thread.h> -#include <vespa/vespalib/util/sync.h> namespace storage::framework::defaultimplementation { @@ -14,13 +13,13 @@ struct ThreadPoolImpl : public ThreadPool { FastOS_ThreadPool _backendThreadPool; std::vector<ThreadImpl*> _threads; - vespalib::Lock _threadVectorLock; + mutable std::mutex _threadVectorLock; Clock & _clock; bool _stopping; public: ThreadPoolImpl(Clock&); - ~ThreadPoolImpl(); + ~ThreadPoolImpl() override; Thread::UP startThread(Runnable&, vespalib::stringref id, uint64_t waitTimeMs, uint64_t maxProcessTime, int ticksBeforeWait) override; diff --git a/storageframework/src/vespa/storageframework/generic/component/component.cpp b/storageframework/src/vespa/storageframework/generic/component/component.cpp index b0569bdee41..3818a2865d9 100644 --- a/storageframework/src/vespa/storageframework/generic/component/component.cpp +++ b/storageframework/src/vespa/storageframework/generic/component/component.cpp @@ -4,6 +4,7 @@ #include "componentregister.h" #include <vespa/storageframework/generic/metric/metricregistrator.h> #include <vespa/storageframework/generic/thread/threadpool.h> +#include <cassert> namespace storage::framework { diff --git a/storageframework/src/vespa/storageframework/generic/component/component.h b/storageframework/src/vespa/storageframework/generic/component/component.h index 165d9c47d6e..c91d7feb532 100644 --- a/storageframework/src/vespa/storageframework/generic/component/component.h +++ b/storageframework/src/vespa/storageframework/generic/component/component.h @@ -110,7 +110,6 @@ class Component : private ManagedComponent void setClock(Clock& c) override { _clock = &c; } void setThreadPool(ThreadPool& tp) override { _threadPool = &tp; } void setUpgradeFlag(UpgradeFlags flag) override { - assert(_upgradeFlag.is_lock_free()); _upgradeFlag.store(flag, std::memory_order_relaxed); } void open() override; diff --git a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp index dc4f8bbc1f1..3178220654e 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp +++ b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp @@ -1,8 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tickingthread.h" #include "threadpool.h" -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/sync.h> +#include <cassert> namespace storage::framework { diff --git a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h index 477c17b1cc3..1ed02d0f6fd 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h +++ b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h @@ -21,7 +21,6 @@ #include <memory> #include <vespa/storageframework/generic/clock/time.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/sync.h> namespace storage::framework { diff --git a/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp b/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp index d51bd57e942..13b396f5284 100644 --- a/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp @@ -30,7 +30,7 @@ struct SubFieldTerm vespalib::string _field_name; const QueryTerm* _term; public: - SubFieldTerm(vespalib::string field_name, const QueryTerm* term) + SubFieldTerm(vespalib::string field_name, const QueryTerm* term) noexcept : _field_name(std::move(field_name)), _term(term) { diff --git a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp index 107a1b44208..614d11aabfa 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp @@ -57,7 +57,7 @@ SearchEnvironment::SearchEnvironment(const config::ConfigUri & configUri) : SearchEnvironment::~SearchEnvironment() { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _threadLocals.clear(); } @@ -68,12 +68,12 @@ SearchEnvironment::getEnv(const vespalib::string & searchCluster) if (_localEnvMap == nullptr) { EnvMapUP envMap = std::make_unique<EnvMap>(); _localEnvMap = envMap.get(); - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _threadLocals.emplace_back(std::move(envMap)); } EnvMap::iterator localFound = _localEnvMap->find(searchCluster); if (localFound == _localEnvMap->end()) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); EnvMap::iterator found = _envMap.find(searchCluster); if (found == _envMap.end()) { LOG(debug, "Init VSMAdapter with config id = '%s'", searchCluster.c_str()); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.h b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.h index 126b8370105..3dce03e9402 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.h @@ -19,7 +19,7 @@ private: public: typedef std::shared_ptr<Env> SP; Env(const vespalib::string & muffens, const config::ConfigUri & configUri, Fast_NormalizeWordFolder & wf); - ~Env(); + ~Env() override; const vsm::VSMAdapter * getVSMAdapter() const { return _vsmAdapter.get(); } const RankManager * getRankManager() const { return _rankManager.get(); } void configure(const config::ConfigSnapshot & snapshot) override; @@ -38,7 +38,7 @@ private: static __thread EnvMap * _localEnvMap; EnvMap _envMap; ThreadLocals _threadLocals; - vespalib::Lock _lock; + std::mutex _lock; Fast_NormalizeWordFolder _wordFolder; config::ConfigUri _configUri; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 9f1887c3b53..eb3879863a5 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -57,7 +57,7 @@ private: * @param fid the field id of the attribute field. * @param attr a guard to the attribute vector. **/ - AttrInfo(vsm::FieldIdT fid, search::AttributeGuard::UP attr) : + AttrInfo(vsm::FieldIdT fid, search::AttributeGuard::UP attr) noexcept : _field(fid), _ascending(true), _converter(nullptr), @@ -71,7 +71,7 @@ private: * @param ascending whether this attribute should be sorted ascending or not. * @param converter is a converter to apply to the attribute before sorting. **/ - AttrInfo(vsm::FieldIdT fid, search::AttributeGuard::UP attr, bool ascending, const search::common::BlobConverter * converter) : + AttrInfo(vsm::FieldIdT fid, search::AttributeGuard::UP attr, bool ascending, const search::common::BlobConverter * converter) noexcept : _field(fid), _ascending(ascending), _converter(converter), diff --git a/vbench/src/apps/vbench/vbench.cpp b/vbench/src/apps/vbench/vbench.cpp index 61206c772e7..ab695f0b728 100644 --- a/vbench/src/apps/vbench/vbench.cpp +++ b/vbench/src/apps/vbench/vbench.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/programoptions.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/thread.h> #include <vespa/vespalib/util/runnable_pair.h> #include <vbench/vbench/vbench.h> diff --git a/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp b/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp index c0c5982d22b..08d44057dfc 100644 --- a/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp +++ b/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vbench/test/all.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/net/crypto_engine.h> using namespace vbench; diff --git a/vbench/src/tests/time_queue/time_queue_test.cpp b/vbench/src/tests/time_queue/time_queue_test.cpp index 8b683f99127..5ce4feab747 100644 --- a/vbench/src/tests/time_queue/time_queue_test.cpp +++ b/vbench/src/tests/time_queue/time_queue_test.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vbench/test/all.h> -#include <vespa/vespalib/util/sync.h> using namespace vbench; diff --git a/vbench/src/vbench/core/dispatcher.h b/vbench/src/vbench/core/dispatcher.h index 212cf04a06e..21db6c931a4 100644 --- a/vbench/src/vbench/core/dispatcher.h +++ b/vbench/src/vbench/core/dispatcher.h @@ -5,7 +5,6 @@ #include "handler.h" #include "provider.h" #include "closeable.h" -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/gate.h> #include <vector> @@ -31,13 +30,13 @@ private: }; Handler<T> &_fallback; - vespalib::Lock _lock; + mutable std::mutex _lock; std::vector<ThreadState*> _threads; bool _closed; public: - Dispatcher(Handler<T> &fallback); - ~Dispatcher(); + explicit Dispatcher(Handler<T> &fallback); + ~Dispatcher() override; bool waitForThreads(size_t threads, size_t pollCnt) const; void close() override; void handle(std::unique_ptr<T> obj) override; diff --git a/vbench/src/vbench/core/dispatcher.hpp b/vbench/src/vbench/core/dispatcher.hpp index 9fd5c5dd527..9a985797967 100644 --- a/vbench/src/vbench/core/dispatcher.hpp +++ b/vbench/src/vbench/core/dispatcher.hpp @@ -14,7 +14,7 @@ Dispatcher<T>::Dispatcher(Handler<T> &fallback) } template <typename T> -Dispatcher<T>::~Dispatcher() {} +Dispatcher<T>::~Dispatcher() = default; template <typename T> bool @@ -25,7 +25,7 @@ Dispatcher<T>::waitForThreads(size_t threads, size_t pollCnt) const vespalib::Thread::sleep(20); } { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); if (_threads.size() >= threads) { return true; } @@ -40,7 +40,7 @@ Dispatcher<T>::close() { std::vector<ThreadState*> threads; { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); std::swap(_threads, threads); _closed = true; } @@ -53,7 +53,7 @@ template <typename T> void Dispatcher<T>::handle(std::unique_ptr<T> obj) { - vespalib::LockGuard guard(_lock); + std::unique_lock guard(_lock); if (!_threads.empty()) { ThreadState *state = _threads.back(); _threads.pop_back(); @@ -75,7 +75,7 @@ Dispatcher<T>::provide() { ThreadState state; { - vespalib::LockGuard guard(_lock); + std::unique_lock guard(_lock); if (!_closed) { _threads.push_back(&state); guard.unlock(); diff --git a/vbench/src/vbench/core/time_queue.h b/vbench/src/vbench/core/time_queue.h index 2af6f3edbcd..5237a6bc0fb 100644 --- a/vbench/src/vbench/core/time_queue.h +++ b/vbench/src/vbench/core/time_queue.h @@ -21,14 +21,14 @@ private: struct Entry { std::unique_ptr<T> object; double time; - Entry(std::unique_ptr<T> obj, double t) : object(std::move(obj)), time(t) {} - Entry(Entry &&rhs) : object(std::move(rhs.object)), time(rhs.time) {} - Entry &operator=(Entry &&rhs) { + Entry(std::unique_ptr<T> obj, double t) noexcept : object(std::move(obj)), time(t) {} + Entry(Entry &&rhs) noexcept : object(std::move(rhs.object)), time(rhs.time) {} + Entry &operator=(Entry &&rhs) noexcept { object = std::move(rhs.object); time = rhs.time; return *this; } - bool operator<(const Entry &rhs) const { + bool operator<(const Entry &rhs) const noexcept { return (time < rhs.time); } }; @@ -42,6 +42,7 @@ private: public: TimeQueue(double window, double tick); + ~TimeQueue(); void close() override; void discard(); void insert(std::unique_ptr<T> obj, double time); diff --git a/vbench/src/vbench/core/time_queue.hpp b/vbench/src/vbench/core/time_queue.hpp index 4a70e258935..0a1a74d72db 100644 --- a/vbench/src/vbench/core/time_queue.hpp +++ b/vbench/src/vbench/core/time_queue.hpp @@ -13,6 +13,9 @@ TimeQueue<T>::TimeQueue(double window, double tick) { } +template<typename T> +TimeQueue<T>::~TimeQueue() = default; + template <typename T> void TimeQueue<T>::close() diff --git a/vbench/src/vbench/http/http_connection_pool.cpp b/vbench/src/vbench/http/http_connection_pool.cpp index ffe6518e374..b949279b7d3 100644 --- a/vbench/src/vbench/http/http_connection_pool.cpp +++ b/vbench/src/vbench/http/http_connection_pool.cpp @@ -19,7 +19,7 @@ HttpConnection::UP HttpConnectionPool::getConnection(const ServerSpec &server) { double now = _timer.sample(); - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); auto res = _map.insert(std::make_pair(server, _store.size())); if (res.second) { _store.emplace_back(); @@ -40,7 +40,7 @@ void HttpConnectionPool::putConnection(HttpConnection::UP conn) { double now = _timer.sample(); - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); conn->touch(now); size_t idx = _map[conn->server()]; assert(idx < _store.size()); diff --git a/vbench/src/vbench/http/http_connection_pool.h b/vbench/src/vbench/http/http_connection_pool.h index 5dd9eb6361c..f5c279e7efe 100644 --- a/vbench/src/vbench/http/http_connection_pool.h +++ b/vbench/src/vbench/http/http_connection_pool.h @@ -5,7 +5,6 @@ #include "http_connection.h" #include <vbench/core/timer.h> #include <vespa/vespalib/util/arrayqueue.hpp> -#include <vespa/vespalib/util/sync.h> #include <map> namespace vbench { @@ -22,7 +21,7 @@ private: typedef std::map<ServerSpec, size_t> Map; using CryptoEngine = vespalib::CryptoEngine; - vespalib::Lock _lock; + std::mutex _lock; Map _map; std::vector<Queue> _store; CryptoEngine::SP _crypto; diff --git a/vbench/src/vbench/vbench/request_scheduler.h b/vbench/src/vbench/vbench/request_scheduler.h index 2f9e9177c53..69be0bba6ca 100644 --- a/vbench/src/vbench/vbench/request_scheduler.h +++ b/vbench/src/vbench/vbench/request_scheduler.h @@ -7,7 +7,6 @@ #include <vbench/core/time_queue.h> #include <vbench/core/dispatcher.h> #include <vbench/core/handler_thread.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/active.h> namespace vbench { diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index b997bf5f983..3c1ee756e56 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -21,7 +21,6 @@ vespa_define_module( TESTS src/tests - src/tests/bucketdistribution src/tests/container src/tests/distribution src/tests/state diff --git a/vdslib/src/tests/CMakeLists.txt b/vdslib/src/tests/CMakeLists.txt index bc230a7157b..6cf1ba5e33f 100644 --- a/vdslib/src/tests/CMakeLists.txt +++ b/vdslib/src/tests/CMakeLists.txt @@ -6,7 +6,6 @@ vespa_add_executable(vdslib_gtest_runner_app TEST SOURCES gtest_runner.cpp DEPENDS - vdslib_bucketdistributiontest vdslib_containertest vdslib_testdistribution vdslib_teststate diff --git a/vdslib/src/tests/bucketdistribution/.gitignore b/vdslib/src/tests/bucketdistribution/.gitignore deleted file mode 100644 index 583460ae288..00000000000 --- a/vdslib/src/tests/bucketdistribution/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.So -.depend -Makefile diff --git a/vdslib/src/tests/bucketdistribution/CMakeLists.txt b/vdslib/src/tests/bucketdistribution/CMakeLists.txt deleted file mode 100644 index 79ca63b72f2..00000000000 --- a/vdslib/src/tests/bucketdistribution/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(vdslib_bucketdistributiontest - SOURCES - bucketdistributiontest.cpp - DEPENDS - vdslib - GTest::GTest -) diff --git a/vdslib/src/tests/bucketdistribution/bucketdistributiontest.cpp b/vdslib/src/tests/bucketdistribution/bucketdistributiontest.cpp deleted file mode 100644 index 7f2b012680c..00000000000 --- a/vdslib/src/tests/bucketdistribution/bucketdistributiontest.cpp +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdslib/bucketdistribution.h> -#include <gtest/gtest.h> - -using namespace vdslib; - -void -assertDistribution(uint32_t numColumns, uint32_t numBucketBits, const uint32_t expected[]) -{ - BucketDistribution bd(numColumns, numBucketBits); - EXPECT_EQ(numColumns, bd.getNumColumns()); - EXPECT_EQ((uint32_t)(1 << numBucketBits), bd.getNumBuckets()); - for (uint32_t i = 0; i < bd.getNumBuckets(); ++i) { - EXPECT_EQ(expected[i], bd.getColumn(document::BucketId(16, i))); - } -} - -TEST(BucketDistributionTest, testDistribution) -{ - const uint32_t expected4[] = { - 10, 11, 9, 6, 4, 8, 14, 1, 13, 2, 12, 3, 5, 7, 15, 0 }; - assertDistribution(16, 4, expected4); - - const uint32_t expected5[] = { - 11, 12, 11, 13, 8, 13, 8, 9, 4, 5, 6, 12, 10, 15, 1, 1, 7, 9, 14, 2, 2, 14, 3, 3, 4, 5, 6, 7, 10, 15, 0, 0 }; - assertDistribution(16, 5, expected5); - - const uint32_t expected6[] = { - 13, 13, 13, 13, 9, 11, 8, 9, 11, 14, 9, 11, 14, 14, 8, 10, 11, 14, 4, 5, 5, 6, 6, 7, 8, 10, 12, 15, 1, 1, 1, 1, - 6, 7, 8, 10, 12, 15, 2, 2, 2, 2, 12, 15, 3, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7, 7, 9, 10, 12, 15, 0, 0, 0, 0 }; - assertDistribution(16, 6, expected6); - - const uint32_t expected7[] = { - 14, 14, 14, 13, 11, 14, 14, 10, 13, 14, 10, 12, 8, 8, 9, 10, 9, 10, 11, 12, 13, 15, 11, 12, 13, 13, 15, 8, 8, 9, 10, 11, - 11, 12, 13, 15, 4, 4, 5, 5, 5, 5, 15, 6, 6, 7, 7, 7, 8, 9, 9, 10, 11, 12, 14, 15, 1, 1, 1, 1, 1, 1, 1, 1, - 6, 6, 6, 7, 7, 8, 8, 9, 10, 11, 12, 13, 15, 2, 2, 2, 2, 2, 2, 2, 2, 12, 13, 15, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 9, 9, 10, 11, 12, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0 }; - assertDistribution(16, 7, expected7); - - const uint32_t expected8[] = { - 15, 14, 15, 15, 12, 14, 15, 12, 12, 11, 12, 13, 14, 13, 13, 13, 14, 15, 15, 9, 14, 9, 15, 10, 11, 11, 12, 8, 8, 8, 8, 9, - 9, 10, 10, 10, 11, 11, 12, 13, 13, 14, 10, 11, 11, 12, 13, 13, 14, 13, 13, 14, 15, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, - 10, 10, 11, 11, 12, 13, 13, 14, 15, 4, 4, 4, 4, 15, 5, 5, 5, 5, 5, 5, 5, 15, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, - 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 14, 15, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 14, 15, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 12, 13, 14, 14, 15, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, - 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - assertDistribution(16, 8, expected8); - - const uint32_t expected9[] = { - 15, 15, 15, 15, 14, 14, 15, 13, 13, 13, 13, 13, 14, 14, 15, 12, 14, 15, 15, 11, 11, 12, 13, 13, 13, 14, 14, 15, 15, 10, 12, 12, - 12, 13, 13, 13, 14, 14, 15, 15, 9, 9, 9, 10, 10, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 15, 15, 8, 8, 8, 8, 9, 9, 9, - 9, 9, 9, 10, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 14, 14, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12, - 13, 13, 14, 14, 14, 15, 15, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, - 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 4, 4, 4, 4, 4, 4, 4, 15, 15, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 14, 14, 14, 15, 15, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 15, 15, 15, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, - 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 15, 15, 15, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - assertDistribution(16, 9, expected9); -} - -TEST(BucketDistributionTest, testNumBucketBits) -{ - BucketDistribution bd(1, 4); - for (uint32_t i = 0; i <= 0xf; ++i) { - EXPECT_EQ(0u, bd.getColumn(document::BucketId(32, (rand() << 4) & i))); - } - - bd.reset(); - bd.setNumColumns(1); - bd.setNumBucketBits(8); - for (uint32_t i = 0; i <= 0xff; ++i) { - EXPECT_EQ(0u, bd.getColumn(document::BucketId(32, (rand() << 8) & i))); - } - - bd.reset(); - bd.setNumColumns(1); - bd.setNumBucketBits(16); - for (uint32_t i = 0; i <= 0xffff; ++i) { - EXPECT_EQ(0u, bd.getColumn(document::BucketId(32, (rand() << 16) & i))); - } -} diff --git a/vdslib/src/tests/thread/taskschedulertest.cpp b/vdslib/src/tests/thread/taskschedulertest.cpp index 1925625172c..54877fae62b 100644 --- a/vdslib/src/tests/thread/taskschedulertest.cpp +++ b/vdslib/src/tests/thread/taskschedulertest.cpp @@ -2,7 +2,6 @@ #include <vespa/vdslib/thread/taskscheduler.h> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/util/time.h> #include <thread> namespace vdslib { @@ -10,24 +9,24 @@ namespace vdslib { namespace { struct TestWatch : public TaskScheduler::Watch { - vespalib::Lock _lock; + mutable std::mutex _lock; uint64_t _time; TestWatch(uint64_t startTime = 0) : _time(startTime) {} - ~TestWatch() {} + ~TestWatch() = default; TaskScheduler::Time getTime() const override { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); return _time; } void increment(uint64_t ms) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _time += ms; } void set(uint64_t ms) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); _time = ms; } }; diff --git a/vdslib/src/vespa/vdslib/CMakeLists.txt b/vdslib/src/vespa/vdslib/CMakeLists.txt index ace44439a23..cf5053a5ceb 100644 --- a/vdslib/src/vespa/vdslib/CMakeLists.txt +++ b/vdslib/src/vespa/vdslib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vdslib SOURCES - bucketdistribution.cpp $<TARGET_OBJECTS:vdslib_container> $<TARGET_OBJECTS:vdslib_state> $<TARGET_OBJECTS:vdslib_distribution> diff --git a/vdslib/src/vespa/vdslib/bucketdistribution.cpp b/vdslib/src/vespa/vdslib/bucketdistribution.cpp deleted file mode 100644 index a8c39ad6577..00000000000 --- a/vdslib/src/vespa/vdslib/bucketdistribution.cpp +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "bucketdistribution.h" - -#include <vespa/log/log.h> -LOG_SETUP(".bucketdistribution"); - -namespace vdslib { - -BucketDistribution::BucketDistribution(uint32_t numColumns, uint32_t numBucketBits) : - _numColumns(0), - _numBucketBits(numBucketBits), - _bucketToColumn(), - _lock() -{ - _bucketToColumn.resize(getNumBuckets()); - reset(); - setNumColumns(numColumns); -} - -BucketDistribution::BucketDistribution(const BucketDistribution &) = default; -BucketDistribution & BucketDistribution::operator = (const BucketDistribution &) = default; - -BucketDistribution::~BucketDistribution() {} - -void -BucketDistribution::getBucketCount(uint32_t numColumns, uint32_t numBucketBits, std::vector<uint32_t> &ret) -{ - ret.resize(numColumns); - uint32_t cnt = getNumBuckets(numBucketBits) / numColumns; - uint32_t rst = getNumBuckets(numBucketBits) % numColumns; - for (uint32_t i = 0; i < numColumns; ++i) { - ret[i] = cnt + (i < rst ? 1 : 0); - } -} - -void -BucketDistribution::getBucketMigrateCount(uint32_t numColumns, uint32_t numBucketBits, std::vector<uint32_t> &ret) -{ - getBucketCount(numColumns++, numBucketBits, ret); - uint32_t cnt = getNumBuckets(numBucketBits) / numColumns; - uint32_t rst = getNumBuckets(numBucketBits) % numColumns; - for (uint32_t i = 0; i < numColumns - 1; ++i) { - ret[i] -= cnt + (i < rst ? 1 : 0); - } -} - -void -BucketDistribution::reset() -{ - for (std::vector<uint32_t>::iterator it = _bucketToColumn.begin(); - it != _bucketToColumn.end(); ++it) { - *it = 0; - } - _numColumns = 1; -} - -void -BucketDistribution::addColumn() -{ - uint32_t newColumns = _numColumns + 1; - std::vector<uint32_t> migrate; - getBucketMigrateCount(_numColumns, _numBucketBits, migrate); - uint32_t numBuckets = getNumBuckets(_numBucketBits); - for (uint32_t i = 0; i < numBuckets; ++i) { - uint32_t old = _bucketToColumn[i]; - if (migrate[old] > 0) { - _bucketToColumn[i] = _numColumns; // move this bucket to the new column - migrate[old]--; - } - } - _numColumns = newColumns; -} - -void -BucketDistribution::setNumColumns(uint32_t numColumns) -{ - vespalib::LockGuard guard(_lock); - if (numColumns < _numColumns) { - reset(); - } - if (numColumns == _numColumns) { - return; - } - for (int i = numColumns - _numColumns; --i >= 0; ) { - addColumn(); - } -} - -void -BucketDistribution::setNumBucketBits(uint32_t numBucketBits) -{ - uint32_t numColumns; - { - vespalib::LockGuard guard(_lock); - if (numBucketBits == _numBucketBits) { - return; - } - _numBucketBits = numBucketBits; - _bucketToColumn.resize(getNumBuckets(numBucketBits)); - numColumns = _numColumns; - reset(); - } - setNumColumns(numColumns); -} - -uint32_t -BucketDistribution::getColumn(const document::BucketId &bucketId) const -{ - uint32_t ret = (uint32_t)(bucketId.getId() & (getNumBuckets(_numBucketBits) - 1)); - if (ret >= _bucketToColumn.size()) { - LOG(error, - "The bucket distribution map is not in sync with the number of bucket bits. " - "This should never happen! Distribution is broken!!"); - return 0; - } - return _bucketToColumn[ret]; -} - -} diff --git a/vdslib/src/vespa/vdslib/bucketdistribution.h b/vdslib/src/vespa/vdslib/bucketdistribution.h deleted file mode 100644 index c9e584af7ef..00000000000 --- a/vdslib/src/vespa/vdslib/bucketdistribution.h +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/document/bucket/bucketid.h> -#include <vespa/vespalib/util/sync.h> -#include <vector> - -namespace vdslib { - -/** - * Stable algorithmic hash distribution; this class assigns hash buckets to targets. The number of hash buckets should - * be large compared to the number of targets. The mapping from hash value to hash bucket is performed outside this - * class. - */ -class BucketDistribution { -public: - /** - * Constructs a new bucket distribution object with a given number of columns and buckets. - * - * @param numColumns The number of columns to distribute to. - * @param numBucketBits The number of bits to use for bucket id. - */ - BucketDistribution(uint32_t numColumns, uint32_t numBucketBits); - BucketDistribution(const BucketDistribution &); - BucketDistribution & operator = (const BucketDistribution &); - BucketDistribution(BucketDistribution &&) = default; - BucketDistribution & operator = (BucketDistribution &&) = default; - ~BucketDistribution(); - - /** - * Returns the number of buckets that the given number of bucket bits will allow. - * - * @param numBucketBits The number of bits to use for bucket id. - * @return The number of buckets allowed. - */ - static uint32_t getNumBuckets(uint32_t numBucketBits) { return 1 << numBucketBits; } - - /** - * This method returns a list that contains the distributions of the given number of buckets over the given number - * of columns. - * - * @param numColumns The number of columns to distribute to. - * @param numBucketBits The number of bits to use for bucket id. - * @param ret List to fill with the bucket distribution. - */ - static void getBucketCount(uint32_t numColumns, uint32_t numBucketBits, std::vector<uint32_t> &ret); - - /** - * This method returns a list similar to {@link this#getBucketCount(int,int)}, except that the returned list - * contains the number of buckets that will have to be migrated from each column if an additional column was added. - * - * @param numColumns The original number of columns. - * @param numBucketBits The number of bits to use for bucket id. - * @param ret List to fill with the number of buckets to migrate, one value per column. - */ - static void getBucketMigrateCount(uint32_t numColumns, uint32_t numBucketBits, std::vector<uint32_t> &ret); - - /** - * Sets the number of columns to distribute to to 1, and resets the content of the internal bucket-to-column map so - * that it all buckets point to that single column. - */ - void reset(); - - /** - * Sets the number of columns to use for this document distribution object. This will reset and setup this object - * from scratch. The original number of buckets is maintained. - * - * @param numColumns The new number of columns to distribute to. - */ - void setNumColumns(uint32_t numColumns); - - /** - * Returns the number of columns to distribute to. - * - * @return The number of columns. - */ - uint32_t getNumColumns() const { return _numColumns; } - - /** - * Sets the number of buckets to use for this document distribution object. This will reset and setup this object - * from scratch. The original number of columns is maintained. - * - * @param numBucketBits The new number of bits to use for bucket id. - */ - void setNumBucketBits(uint32_t numBucketBits); - - /** - * Returns the number of bits used for bucket identifiers. - * - * @return The number of bits. - */ - uint32_t getNumBucketBits() const { return _numBucketBits; } - - /** - * Returns the number of buckets available using the configured number of bucket bits. - * - * @return The number of buckets. - */ - uint32_t getNumBuckets() const { return getNumBuckets(_numBucketBits); } - - /** - * This method maps the given bucket id to its corresponding column. - * - * @param bucketId The bucket whose column to lookup. - * @return The column to distribute the bucket to. - */ - uint32_t getColumn(const document::BucketId &bucketId) const; - -private: - /** - * Adds a single column to this bucket distribution object. This will modify the internal bucket-to-column map so - * that it takes into account the new column. - */ - void addColumn(); - -private: - uint32_t _numColumns; // The number of columns to distribute to. - uint32_t _numBucketBits; // The number of bits to use for bucket identification. - std::vector<uint32_t> _bucketToColumn; // A map from bucket id to column index. - vespalib::Lock _lock; -}; - -} diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index 0a154f3dec1..474ed63e8c3 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -14,6 +14,7 @@ #include <list> #include <algorithm> #include <cmath> +#include <cassert> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".vdslib.distribution"); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/DocumentOperationExecutorImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/DocumentOperationExecutorImpl.java index 7074d363316..309bed38c0b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/DocumentOperationExecutorImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/DocumentOperationExecutorImpl.java @@ -40,6 +40,7 @@ import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -85,6 +86,7 @@ public class DocumentOperationExecutorImpl implements DocumentOperationExecutor private final DelayQueue throttled; private final DelayQueue timeouts; private final Map<VisitorControlHandler, VisitorSession> visits = new ConcurrentHashMap<>(); + private final ExecutorService visitSessionShutdownExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("visit-session-shutdown-")); public DocumentOperationExecutorImpl(ClusterListConfig clustersConfig, AllClustersBucketSpacesConfig bucketsConfig, DocumentOperationExecutorConfig executorConfig, DocumentAccess access, Clock clock) { @@ -105,7 +107,7 @@ public class DocumentOperationExecutorImpl implements DocumentOperationExecutor this.asyncSession = access.createAsyncSession(new AsyncParameters()); this.clock = requireNonNull(clock); this.clusters = Map.copyOf(clusters); - this.throttled = new DelayQueue(maxThrottled, this::send, resendDelay, clock, "throttle"); + this.throttled = new DelayQueue(maxThrottled, this::send, Duration.ZERO, clock, "throttle"); this.timeouts = new DelayQueue(Long.MAX_VALUE, (__, context) -> { context.error(TIMEOUT, "Timed out after " + defaultTimeout); return true; @@ -150,14 +152,16 @@ public class DocumentOperationExecutorImpl implements DocumentOperationExecutor @Override public void shutdown() { long shutdownMillis = clock.instant().plusSeconds(20).toEpochMilli(); + visitSessionShutdownExecutor.shutdown(); visits.values().forEach(VisitorSession::destroy); Future<?> throttleShutdown = throttled.shutdown(Duration.ofSeconds(10), context -> context.error(OVERLOAD, "Retry on overload failed due to shutdown")); Future<?> timeoutShutdown = timeouts.shutdown(Duration.ofSeconds(15), context -> context.error(TIMEOUT, "Timed out due to shutdown")); try { - throttleShutdown.get(Math.max(0, shutdownMillis - clock.millis()), TimeUnit.MILLISECONDS); - timeoutShutdown.get(Math.max(0, shutdownMillis - clock.millis()), TimeUnit.MILLISECONDS); + throttleShutdown.get(Math.max(1, shutdownMillis - clock.millis()), TimeUnit.MILLISECONDS); + timeoutShutdown.get(Math.max(1, shutdownMillis - clock.millis()), TimeUnit.MILLISECONDS); + visitSessionShutdownExecutor.awaitTermination(Math.max(1, shutdownMillis - clock.millis()), TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { throttleShutdown.cancel(true); @@ -212,12 +216,18 @@ public class DocumentOperationExecutorImpl implements DocumentOperationExecutor context.error(ERROR, message != null ? message : "Visiting failed"); } done.set(true); // This may be reached before dispatching thread is done putting us in the map. - visits.computeIfPresent(this, (__, session) -> { session.destroy(); return null; }); + visits.computeIfPresent(this, (__, session) -> { + visitSessionShutdownExecutor.execute(() -> session.destroy()); + return null; + }); } }); visits.put(parameters.getControlHandler(), access.createVisitorSession(parameters)); if (done.get()) - visits.computeIfPresent(parameters.getControlHandler(), (__, session) -> { session.destroy(); return null; }); + visits.computeIfPresent(parameters.getControlHandler(), (__, session) -> { + visitSessionShutdownExecutor.execute(() -> session.destroy()); + return null; + }); } catch (IllegalArgumentException | ParseException e) { context.error(BAD_REQUEST, Exceptions.toMessageString(e)); @@ -386,7 +396,7 @@ public class DocumentOperationExecutorImpl implements DocumentOperationExecutor synchronized (this) { do { notify(); - wait(Math.max(0, waitUntilMillis - clock.millis())); + wait(Math.max(1, waitUntilMillis - clock.millis())); } while (clock.millis() < waitUntilMillis); } diff --git a/vespaclient-container-plugin/src/main/resources/configdefinitions/document-operation-executor.def b/vespaclient-container-plugin/src/main/resources/configdefinitions/document-operation-executor.def index 19f4f50648b..770189f90f5 100644 --- a/vespaclient-container-plugin/src/main/resources/configdefinitions/document-operation-executor.def +++ b/vespaclient-container-plugin/src/main/resources/configdefinitions/document-operation-executor.def @@ -11,5 +11,5 @@ defaultTimeoutSeconds int default=180 visitTimeoutSeconds int default=120 # Bound on number of document operations to keep in retry queue — further operations are rejected -maxThrottled int default=200 +maxThrottled int default=1000 diff --git a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp index 082aa83c3da..9393562bf28 100644 --- a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp +++ b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp @@ -254,11 +254,11 @@ struct StateApp : public FastOS_Application { std::unique_ptr<slobrok::api::MirrorAPI> slobrok; if (_options._slobrokConnectionSpec == "") { config::ConfigUri config(_options._slobrokConfigId); - slobrok.reset(new slobrok::api::MirrorAPI(supervisor.supervisor(), config)); + slobrok = std::make_unique<slobrok::api::MirrorAPI>(supervisor.supervisor(), slobrok::ConfiguratorFactory(config)); } else { std::vector<std::string> specList; specList.push_back(_options._slobrokConnectionSpec); - slobrok.reset(new slobrok::api::MirrorAPI(supervisor.supervisor(), specList)); + slobrok = std::make_unique<slobrok::api::MirrorAPI>(supervisor.supervisor(), slobrok::ConfiguratorFactory(specList)); } LOG(debug, "Waiting for slobrok data to be available."); uint64_t startTime = getTimeInMillis(); diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index ec003329999..278d32c118d 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -46,7 +46,6 @@ vespa_define_module( src/tests/datastore/unique_store src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator - src/tests/delegatelist src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache @@ -94,7 +93,6 @@ vespa_define_module( src/tests/regex src/tests/rendezvous src/tests/runnable_pair - src/tests/rwlock src/tests/sha1 src/tests/sharedptr src/tests/signalhandler diff --git a/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp b/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp index 37a22108dc5..6a9215c3eb9 100644 --- a/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp +++ b/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp @@ -40,7 +40,7 @@ struct DictionaryReadTest : public ::testing::Test { { } DictionaryReadTest& add(uint32_t value) { - auto result = dict.add(Comparator(value), [=]() { return EntryRef(value); }); + auto result = dict.add(Comparator(value), [=]() noexcept { return EntryRef(value); }); assert(result.inserted()); return *this; } diff --git a/vespalib/src/tests/delegatelist/.cvsignore b/vespalib/src/tests/delegatelist/.cvsignore deleted file mode 100644 index 0e8f4d0be0b..00000000000 --- a/vespalib/src/tests/delegatelist/.cvsignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -delegatelist_test diff --git a/vespalib/src/tests/delegatelist/.gitignore b/vespalib/src/tests/delegatelist/.gitignore deleted file mode 100644 index 42ac4beb0c3..00000000000 --- a/vespalib/src/tests/delegatelist/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -delegatelist_test -vespalib_delegatelist_test_app diff --git a/vespalib/src/tests/delegatelist/CMakeLists.txt b/vespalib/src/tests/delegatelist/CMakeLists.txt deleted file mode 100644 index 71551474445..00000000000 --- a/vespalib/src/tests/delegatelist/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_delegatelist_test_app TEST - SOURCES - delegatelist.cpp - DEPENDS - vespalib -) -vespa_add_test(NAME vespalib_delegatelist_test_app COMMAND vespalib_delegatelist_test_app) diff --git a/vespalib/src/tests/delegatelist/delegatelist.cpp b/vespalib/src/tests/delegatelist/delegatelist.cpp deleted file mode 100644 index 070864dd85a..00000000000 --- a/vespalib/src/tests/delegatelist/delegatelist.cpp +++ /dev/null @@ -1,824 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/delegatelist.hpp> -#include <vespa/vespalib/util/guard.h> -#include <vespa/fastos/thread.h> -#include <queue> - -#include <vespa/log/log.h> -LOG_SETUP("delegatelist_test"); - -using namespace vespalib; - -//----------------------------------------------------------------------------- - -class Test : public TestApp -{ -public: - void testEmpty(); - void testAdd(); - void testRemove(); - void testOneShot(); - void testMultiSnapshot(); - void testActors(); - void testWaitSnapshots(); - void stressTest(); - int Main() override; -}; - -//----------------------------------------------------------------------------- - -namespace { - -class Handler -{ -private: - int _num; -public: - Handler() : _num(0) {} - void add() { _num++; } - int getNum() { return _num; } -}; - -typedef DelegateList<Handler> DL; - -void multicast(DL &dl) { - for (DL::Snapshot snap(dl) ; snap.valid(); snap.next()) { - snap.get()->add(); - } -} - -void multicast_clear(DL &dl) { - DL::Snapshot snap(dl); - dl.clear(); - for (; snap.valid(); snap.next()) { - snap.get()->add(); - } -} - -//----------------------------------------------------------------------------- - -enum { - CMD_MULTICAST, - CMD_MULTICAST_CLEAR, - CMD_ADD, - CMD_REMOVE, - CMD_CLEAR, - CMD_WAIT_SNAP, - CMD_DO, - CMD_DONE, - CMD_EXIT -}; - -struct Command -{ - DL *dl; - int cmd; - int cnt; - Handler *handler; - Command(DL *dl_, int cmd_, int cnt_, Handler *handler_) - : dl(dl_), cmd(cmd_), cnt(cnt_), handler(handler_) {} - Command(const Command &rhs) - : dl(rhs.dl), cmd(rhs.cmd), cnt(rhs.cnt), handler(rhs.handler) {} - Command &operator=(const Command &rhs) { - memcpy(this, &rhs, sizeof(Command)); - return *this; - } - bool operator==(const Command &rhs) { - return memcmp(this, &rhs, sizeof(Command)) == 0; - } -}; - -Command -cmd_multicast(DL *dl) { - return Command(dl, CMD_MULTICAST, 0, 0); -} - -Command -cmd_multicast_clear(DL *dl) { - return Command(dl, CMD_MULTICAST_CLEAR, 0, 0); -} - -Command -cmd_add(DL *dl, Handler *handler) { - return Command(dl, CMD_ADD, 0, handler); -} - -Command -cmd_remove(DL *dl, Handler *handler) { - return Command(dl, CMD_REMOVE, 0, handler); -} - -Command -cmd_clear(DL *dl) { - return Command(dl, CMD_CLEAR, 0, 0); -} - -Command -cmd_wait_snap(DL *dl) { - return Command(dl, CMD_WAIT_SNAP, 0, 0); -} - -Command -cmd_do(int cnt) { - return Command(0, CMD_DO, cnt, 0); -} - -Command -cmd_done() { - return Command(0, CMD_DONE, 0, 0); -} - -Command -cmd_exit() { - return Command(0, CMD_EXIT, 0, 0); -} - -typedef std::vector<Command> CmdList; -typedef std::pair<Command, int> HistEntry; -typedef std::vector<HistEntry> HistList; - -//----------------------------------------------------------------------------- - -struct History { - Lock lock; - HistList list; - History() : lock(), list() {} - void add(const HistEntry &entry) { - LockGuard guard(lock); - list.push_back(entry); - } -}; - -//----------------------------------------------------------------------------- - -template <typename T> -class Queue { -private: - std::queue<T> _q; - Monitor _cond; - int _waitCnt; - Queue(const Queue &); - Queue &operator=(const Queue &); -public: - Queue() : _q(), _cond(), _waitCnt(0) {} - void enqueue(const T &entry) { - MonitorGuard guard(_cond); - _q.push(entry); - if (_waitCnt > 0) { - guard.signal(); - } - } - T dequeue() { - MonitorGuard guard(_cond); - CounterGuard cntGuard(_waitCnt); - while (_q.empty()) { - guard.wait(); - } - T tmp = _q.front(); - _q.pop(); - return tmp; - } - size_t size() const { return _q.size(); } -}; - -typedef Queue<CmdList> CmdListQueue; - -//----------------------------------------------------------------------------- - -class Actor : public FastOS_Runnable -{ -public: - enum { - STATE_INIT, - STATE_IDLE, - STATE_BUSY, - STATE_DONE - }; -private: - int _id; - History *_hist; - CmdListQueue _queue; - Monitor _cond; - int _state; - int _waitCnt; - int _opCnt; - bool _exit; - Actor(const Actor &); - Actor &operator=(const Actor &); - void setState(int state, MonitorGuard &guard); - void doneOp(const Command &cmd); - int perform(int cnt, int start, const CmdList &cmdList); -public: - Actor(int id, History *hist); - ~Actor(); - int getOpCnt() const { return _opCnt; } - int getState() const { return _state; } - void doIt(const CmdList &cmdList); - void doIt(const Command &cmd); - void waitState(int state); - void Run(FastOS_ThreadInterface *, void *) override; -}; - -Actor::Actor(int id, History *hist) - : _id(id), _hist(hist), _queue(), _cond(), _state(STATE_INIT), - _waitCnt(0), _opCnt(0), _exit(false) -{} -Actor::~Actor() {} - -void -Actor::setState(int state, MonitorGuard &guard) { - _state = state; - if (_waitCnt > 0) { - guard.broadcast(); - } -} - - -void -Actor::doneOp(const Command &cmd) -{ - ++_opCnt; - if (_hist != 0) { - _hist->add(HistEntry(cmd, _id)); - } -} - - -int -Actor::perform(int cnt, int start, const CmdList &cmdList) -{ - int doneIdx = cmdList.size(); - for (int i = 0; i < cnt; ++i) { - for (uint32_t idx = start; idx < cmdList.size(); ++idx) { - Command cmd = cmdList[idx]; - switch (cmd.cmd) { - case CMD_MULTICAST: - multicast(*cmd.dl); - doneOp(cmd); - break; - case CMD_MULTICAST_CLEAR: - multicast_clear(*cmd.dl); - doneOp(cmd); - break; - case CMD_ADD: - cmd.dl->add(cmd.handler); - doneOp(cmd); - break; - case CMD_REMOVE: - cmd.dl->remove(cmd.handler); - doneOp(cmd); - break; - case CMD_CLEAR: - cmd.dl->clear(); - doneOp(cmd); - break; - case CMD_WAIT_SNAP: - cmd.dl->waitSnapshots(); - doneOp(cmd); - break; - case CMD_DO: - idx = perform(cmd.cnt, idx + 1, cmdList); - break; - case CMD_DONE: - doneIdx = idx; - idx = cmdList.size(); - break; - case CMD_EXIT: - _exit = true; - return cmdList.size(); - break; - default: - LOG_ABORT("should not be reached"); // that does not seem to work - } - } - } - return doneIdx; -} - - -void -Actor::doIt(const CmdList &cmdList) -{ - MonitorGuard guard(_cond); - setState(STATE_BUSY, guard); - _queue.enqueue(cmdList); -} - - -void -Actor::doIt(const Command &cmd) -{ - CmdList cmdList; - cmdList.push_back(cmd); - doIt(cmdList); -} - - -void -Actor::waitState(int state) { - MonitorGuard guard(_cond); - CounterGuard cntGuard(_waitCnt); - while (_state != state) { - guard.wait(); - } -} - - -void -Actor::Run(FastOS_ThreadInterface *, void *) -{ - while (!_exit) { - { - MonitorGuard guard(_cond); - if (_queue.size() == 0) { - setState(STATE_IDLE, guard); - } - } - CmdList cmdList = _queue.dequeue(); - perform(1, 0, cmdList); - } - { - MonitorGuard guard(_cond); - setState(STATE_DONE, guard); - } -} - -} // namespace <unnamed> - -//----------------------------------------------------------------------------- - -void -Test::testEmpty() -{ - DL multicaster; - multicast(multicaster); - multicast_clear(multicaster); - DL::Snapshot empty_snap(multicaster); - EXPECT_TRUE(!empty_snap.valid()); -} - - -void -Test::testAdd() -{ - DL multicaster; - Handler h1; - Handler h2; - Handler h3; - Handler h4; - Handler h5; - - // ensure correct initial state - EXPECT_TRUE(h1.getNum() == 0); - EXPECT_TRUE(h2.getNum() == 0); - EXPECT_TRUE(h3.getNum() == 0); - EXPECT_TRUE(h4.getNum() == 0); - EXPECT_TRUE(h5.getNum() == 0); - - // test basic adding - multicaster.add(&h1); - multicast(multicaster); - multicaster.add(&h2); - multicast(multicaster); - multicaster.add(&h3); - multicast(multicaster); - multicaster.add(&h4); - multicast(multicaster); - multicaster.add(&h5); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 5); - EXPECT_TRUE(h2.getNum() == 4); - EXPECT_TRUE(h3.getNum() == 3); - EXPECT_TRUE(h4.getNum() == 2); - EXPECT_TRUE(h5.getNum() == 1); - - // duplicate adds - multicaster.add(&h1); - multicaster.add(&h1); - multicaster.add(&h1); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 6); - EXPECT_TRUE(h2.getNum() == 5); - EXPECT_TRUE(h3.getNum() == 4); - EXPECT_TRUE(h4.getNum() == 3); - EXPECT_TRUE(h5.getNum() == 2); -} - - -void -Test::testRemove() -{ - DL multicaster; - Handler h1; - Handler h2; - Handler h3; - Handler h4; - Handler h5; - - multicaster.add(&h1).add(&h2).add(&h3).add(&h4).add(&h5); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 1); - EXPECT_TRUE(h2.getNum() == 1); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 1); - EXPECT_TRUE(h5.getNum() == 1); - - // remove middle - multicaster.remove(&h3); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 2); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 2); - EXPECT_TRUE(h5.getNum() == 2); - - // remove head - multicaster.remove(&h1); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 3); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 3); - EXPECT_TRUE(h5.getNum() == 3); - - // remove tail - multicaster.remove(&h5); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 4); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 4); - EXPECT_TRUE(h5.getNum() == 3); - - // duplicate removes - multicaster.remove(&h1).remove(&h3).remove(&h5); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 5); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 5); - EXPECT_TRUE(h5.getNum() == 3); - - // remove all - multicaster.clear(); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 5); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 5); - EXPECT_TRUE(h5.getNum() == 3); -} - - -void -Test::testOneShot() -{ - DL multicaster; - Handler h1; - Handler h2; - Handler h3; - Handler h4; - Handler h5; - - // oneshot multicast removes handlers - multicaster.add(&h1).add(&h2).add(&h3).add(&h4).add(&h5); - multicast_clear(multicaster); - multicast(multicaster); - EXPECT_TRUE(h1.getNum() == 1); - EXPECT_TRUE(h2.getNum() == 1); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 1); - EXPECT_TRUE(h5.getNum() == 1); -} - - -void -Test::testMultiSnapshot() -{ - DL multicaster; - Handler h1; - Handler h2; - Handler h3; - Handler h4; - Handler h5; - - DL::Snapshot empty_snap(multicaster); - multicaster.add(&h1).add(&h2).add(&h3).add(&h4).add(&h5); - DL::Snapshot snap1(multicaster); - multicaster.remove(&h3); - DL::Snapshot snap2(multicaster); - multicaster.remove(&h1); - DL::Snapshot snap3(multicaster); - multicaster.remove(&h5); - DL::Snapshot snap4(multicaster); - - EXPECT_TRUE(!empty_snap.valid()); - for (; snap1.valid(); snap1.next()) { - snap1.get()->add(); - } - EXPECT_TRUE(h1.getNum() == 1); - EXPECT_TRUE(h2.getNum() == 1); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 1); - EXPECT_TRUE(h5.getNum() == 1); - for (; snap2.valid(); snap2.next()) { - snap2.get()->add(); - } - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 2); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 2); - EXPECT_TRUE(h5.getNum() == 2); - for (; snap3.valid(); snap3.next()) { - snap3.get()->add(); - } - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 3); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 3); - EXPECT_TRUE(h5.getNum() == 3); - for (; snap4.valid(); snap4.next()) { - snap4.get()->add(); - } - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 4); - EXPECT_TRUE(h3.getNum() == 1); - EXPECT_TRUE(h4.getNum() == 4); - EXPECT_TRUE(h5.getNum() == 3); -} - - -void -Test::testActors() -{ - FastOS_ThreadPool pool(65000); - History hist; - Actor a1(1, &hist); - Actor a2(2, &hist); - DL dl; - Handler h1; - Handler h2; - - ASSERT_TRUE(pool.NewThread(&a1, 0) != 0); - ASSERT_TRUE(pool.NewThread(&a2, 0) != 0); - - { - CmdList prog; - prog.push_back(cmd_add(&dl, &h1)); - prog.push_back(cmd_multicast(&dl)); - prog.push_back(cmd_add(&dl, &h2)); - prog.push_back(cmd_multicast(&dl)); - a1.doIt(prog); - a1.waitState(Actor::STATE_IDLE); - } - - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 1); - - { - CmdList prog; - prog.push_back(cmd_remove(&dl, &h1)); - prog.push_back(cmd_multicast(&dl)); - prog.push_back(cmd_clear(&dl)); - prog.push_back(cmd_multicast(&dl)); - a2.doIt(prog); - a2.waitState(Actor::STATE_IDLE); - } - - EXPECT_TRUE(h1.getNum() == 2); - EXPECT_TRUE(h2.getNum() == 2); - - { - CmdList prog; - prog.push_back(cmd_add(&dl, &h1)); - prog.push_back(cmd_add(&dl, &h2)); - prog.push_back(cmd_multicast_clear(&dl)); - prog.push_back(cmd_multicast(&dl)); - a1.doIt(prog); - a1.waitState(Actor::STATE_IDLE); - } - - EXPECT_TRUE(h1.getNum() == 3); - EXPECT_TRUE(h2.getNum() == 3); - - { - CmdList prog; - prog.push_back(cmd_add(&dl, &h1)); - prog.push_back(cmd_add(&dl, &h2)); - prog.push_back(cmd_do(10)); - prog.push_back(cmd_do(10)); - prog.push_back(cmd_multicast(&dl)); - prog.push_back(cmd_done()); - prog.push_back(cmd_done()); - prog.push_back(cmd_exit()); - a2.doIt(prog); - a2.waitState(Actor::STATE_DONE); - } - - EXPECT_TRUE(h1.getNum() == 103); - EXPECT_TRUE(h2.getNum() == 103); - - EXPECT_TRUE(hist.list.size() == 114); - - EXPECT_TRUE(hist.list[0].first == cmd_add(&dl, &h1)); - EXPECT_TRUE(hist.list[1].first == cmd_multicast(&dl)); - EXPECT_TRUE(hist.list[2].first == cmd_add(&dl, &h2)); - EXPECT_TRUE(hist.list[3].first == cmd_multicast(&dl)); - for (int i = 0; i < 4; i++) { - EXPECT_TRUE(hist.list[i].second == 1); - } - - EXPECT_TRUE(hist.list[4].first == cmd_remove(&dl, &h1)); - EXPECT_TRUE(hist.list[5].first == cmd_multicast(&dl)); - EXPECT_TRUE(hist.list[6].first == cmd_clear(&dl)); - EXPECT_TRUE(hist.list[7].first == cmd_multicast(&dl)); - for (int i = 4; i < 8; i++) { - EXPECT_TRUE(hist.list[i].second == 2); - } - - EXPECT_TRUE(hist.list[8].first == cmd_add(&dl, &h1)); - EXPECT_TRUE(hist.list[9].first == cmd_add(&dl, &h2)); - EXPECT_TRUE(hist.list[10].first == cmd_multicast_clear(&dl)); - EXPECT_TRUE(hist.list[11].first == cmd_multicast(&dl)); - for (int i = 8; i < 12; i++) { - EXPECT_TRUE(hist.list[i].second == 1); - } - - EXPECT_TRUE(hist.list[12].first == cmd_add(&dl, &h1)); - EXPECT_TRUE(hist.list[13].first == cmd_add(&dl, &h2)); - EXPECT_TRUE(hist.list[12].second == 2); - EXPECT_TRUE(hist.list[13].second == 2); - - for (int i = 14; i < 114; i++) { - EXPECT_TRUE(hist.list[i].first == cmd_multicast(&dl)); - EXPECT_TRUE(hist.list[i].second == 2); - } - - a1.doIt(cmd_exit()); - a1.waitState(Actor::STATE_DONE); - - EXPECT_TRUE(a1.getOpCnt() == 8); - EXPECT_TRUE(a2.getOpCnt() == 106); -} - - -void -Test::stressTest() -{ - FastOS_ThreadPool pool(65000); - Actor a1(1, 0); - Actor a2(2, 0); - Actor a3(3, 0); - Actor a4(4, 0); - Actor a5(5, 0); - Actor a6(6, 0); - DL dl; - Handler h1; - Handler h2; - Handler h3; - Handler h4; - Handler h5; - int scale = 10000; - - ASSERT_TRUE(pool.NewThread(&a1, 0) != 0); - ASSERT_TRUE(pool.NewThread(&a2, 0) != 0); - ASSERT_TRUE(pool.NewThread(&a3, 0) != 0); - ASSERT_TRUE(pool.NewThread(&a4, 0) != 0); - ASSERT_TRUE(pool.NewThread(&a5, 0) != 0); - ASSERT_TRUE(pool.NewThread(&a6, 0) != 0); - - CmdList prog_multicast; - prog_multicast.push_back(cmd_do(10 * scale)); - prog_multicast.push_back(cmd_multicast(&dl)); - prog_multicast.push_back(cmd_done()); - prog_multicast.push_back(cmd_exit()); - - CmdList prog_wait_snap; - prog_wait_snap.push_back(cmd_do(10 * scale)); - prog_wait_snap.push_back(cmd_wait_snap(&dl)); - prog_wait_snap.push_back(cmd_done()); - prog_wait_snap.push_back(cmd_exit()); - - CmdList prog_add_remove_1; - prog_add_remove_1.push_back(cmd_do(scale)); - prog_add_remove_1.push_back(cmd_add(&dl, &h1)); - prog_add_remove_1.push_back(cmd_add(&dl, &h3)); - prog_add_remove_1.push_back(cmd_remove(&dl, &h2)); - prog_add_remove_1.push_back(cmd_remove(&dl, &h4)); - prog_add_remove_1.push_back(cmd_add(&dl, &h4)); - prog_add_remove_1.push_back(cmd_add(&dl, &h2)); - prog_add_remove_1.push_back(cmd_remove(&dl, &h5)); - prog_add_remove_1.push_back(cmd_remove(&dl, &h3)); - prog_add_remove_1.push_back(cmd_add(&dl, &h5)); - prog_add_remove_1.push_back(cmd_remove(&dl, &h1)); - prog_add_remove_1.push_back(cmd_done()); - prog_add_remove_1.push_back(cmd_exit()); - - CmdList prog_add_remove_2; - prog_add_remove_2.push_back(cmd_do(scale)); - prog_add_remove_2.push_back(cmd_add(&dl, &h5)); - prog_add_remove_2.push_back(cmd_add(&dl, &h4)); - prog_add_remove_2.push_back(cmd_remove(&dl, &h1)); - prog_add_remove_2.push_back(cmd_remove(&dl, &h3)); - prog_add_remove_2.push_back(cmd_add(&dl, &h1)); - prog_add_remove_2.push_back(cmd_remove(&dl, &h2)); - prog_add_remove_2.push_back(cmd_add(&dl, &h2)); - prog_add_remove_2.push_back(cmd_add(&dl, &h3)); - prog_add_remove_2.push_back(cmd_remove(&dl, &h5)); - prog_add_remove_2.push_back(cmd_remove(&dl, &h4)); - prog_add_remove_2.push_back(cmd_done()); - prog_add_remove_2.push_back(cmd_exit()); - - CmdList prog_add_remove_3; - prog_add_remove_3.push_back(cmd_do(scale)); - prog_add_remove_3.push_back(cmd_add(&dl, &h3)); - prog_add_remove_3.push_back(cmd_remove(&dl, &h4)); - prog_add_remove_3.push_back(cmd_remove(&dl, &h3)); - prog_add_remove_3.push_back(cmd_add(&dl, &h5)); - prog_add_remove_3.push_back(cmd_add(&dl, &h2)); - prog_add_remove_3.push_back(cmd_remove(&dl, &h2)); - prog_add_remove_3.push_back(cmd_add(&dl, &h1)); - prog_add_remove_3.push_back(cmd_add(&dl, &h4)); - prog_add_remove_3.push_back(cmd_remove(&dl, &h1)); - prog_add_remove_3.push_back(cmd_remove(&dl, &h5)); - prog_add_remove_3.push_back(cmd_done()); - prog_add_remove_3.push_back(cmd_exit()); - - a1.doIt(prog_multicast); - a2.doIt(prog_multicast); - a3.doIt(prog_wait_snap); - a4.doIt(prog_add_remove_1); - a5.doIt(prog_add_remove_2); - a6.doIt(prog_add_remove_3); - - a1.waitState(Actor::STATE_DONE); - a2.waitState(Actor::STATE_DONE); - a3.waitState(Actor::STATE_DONE); - a4.waitState(Actor::STATE_DONE); - a5.waitState(Actor::STATE_DONE); - a6.waitState(Actor::STATE_DONE); - - EXPECT_TRUE(a1.getOpCnt() == 10 * scale); - EXPECT_TRUE(a2.getOpCnt() == 10 * scale); - EXPECT_TRUE(a3.getOpCnt() == 10 * scale); - EXPECT_TRUE(a4.getOpCnt() == 10 * scale); - EXPECT_TRUE(a5.getOpCnt() == 10 * scale); - EXPECT_TRUE(a6.getOpCnt() == 10 * scale); -} - - -void -Test::testWaitSnapshots() -{ - FastOS_ThreadPool pool(65000); - Actor a1(1, 0); - DL dl; - std::unique_ptr<DL::Snapshot> s1; - std::unique_ptr<DL::Snapshot> s2; - ASSERT_TRUE(pool.NewThread(&a1, 0) != 0); - s1.reset(new DL::Snapshot(dl)); // create snap 1 - a1.doIt(cmd_wait_snap(&dl)); // wait for snaps - std::this_thread::sleep_for(1s); - EXPECT_TRUE(a1.getState() == Actor::STATE_BUSY); // still waiting... - s2.reset(new DL::Snapshot(dl)); // create snap 2 - s1.reset(); // destroy snap 1 - std::this_thread::sleep_for(1s); - EXPECT_TRUE(a1.getState() == Actor::STATE_IDLE); // wait done! - a1.doIt(cmd_exit()); - a1.waitState(Actor::STATE_DONE); - s2.reset(); // destroy snap 2 - EXPECT_TRUE(a1.getOpCnt() == 1); -} - -//----------------------------------------------------------------------------- - -struct Foo { void completeBarrier() {} }; - -int -Test::Main() -{ - TEST_INIT("delegatelist_test"); - LOG(info, "Lock size: %4zu bytes", sizeof(Lock)); - LOG(info, "ArrayQueue size: %4zu bytes", sizeof(ArrayQueue<Foo>)); - LOG(info, "std::vector size: %4zu bytes", sizeof(std::vector<Foo>)); - LOG(info, "EventBarrier size: %4zu bytes", sizeof(EventBarrier<Foo>)); - LOG(info, "DelegateList size: %4zu bytes", sizeof(DelegateList<Foo>)); - - testEmpty(); - testAdd(); - testRemove(); - testOneShot(); - testMultiSnapshot(); - - TEST_FLUSH(); - testActors(); - testWaitSnapshots(); - - TEST_FLUSH(); - stressTest(); - TEST_DONE(); -} - -TEST_APPHOOK(Test); diff --git a/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp b/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp index 6d178093069..71c4a7856c8 100644 --- a/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp +++ b/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp @@ -57,7 +57,7 @@ struct CheckType { }; struct Nop { - void operator()() const {} + void operator()() const noexcept {} }; //----------------------------------------------------------------------------- diff --git a/vespalib/src/tests/executor/blocking_executor_stress.cpp b/vespalib/src/tests/executor/blocking_executor_stress.cpp index b753dc69373..27367f6677c 100644 --- a/vespalib/src/tests/executor/blocking_executor_stress.cpp +++ b/vespalib/src/tests/executor/blocking_executor_stress.cpp @@ -2,7 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/executor.h> -#include <vespa/vespalib/util/sync.h> #include <atomic> using namespace vespalib; @@ -26,7 +25,7 @@ struct MyTask : Executor::Task { size_t size; size_t data; MyTask(size_t size_in) : size(size_in), data(0) {} - virtual void run() override { + void run() override { data += do_stuff(size); ++tasks_run; data += do_stuff(size); diff --git a/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp b/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp index c2a3e66c671..46db619516a 100644 --- a/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp @@ -3,7 +3,6 @@ #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/executor.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/backtrace.h> #include <thread> diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp index 9d69adcd96a..b43a75f8244 100644 --- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp @@ -2,7 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/backtrace.h> #include <atomic> diff --git a/vespalib/src/tests/explore_modern_cpp/explore_modern_cpp_test.cpp b/vespalib/src/tests/explore_modern_cpp/explore_modern_cpp_test.cpp index 1ba368b39a3..29a82138d29 100644 --- a/vespalib/src/tests/explore_modern_cpp/explore_modern_cpp_test.cpp +++ b/vespalib/src/tests/explore_modern_cpp/explore_modern_cpp_test.cpp @@ -5,7 +5,7 @@ TEST("verify how std::function copies lambda closures") { size_t count = 0; size_t value = 0; - auto closure = [count,&value]()mutable{ ++count; value += count; }; + auto closure = [count,&value]() mutable noexcept { ++count; value += count; }; closure(); EXPECT_EQUAL(0u, count); EXPECT_EQUAL(1u, value); // +1 diff --git a/vespalib/src/tests/net/async_resolver/async_resolver_test.cpp b/vespalib/src/tests/net/async_resolver/async_resolver_test.cpp index 434cb8d6a69..f04fe549c09 100644 --- a/vespalib/src/tests/net/async_resolver/async_resolver_test.cpp +++ b/vespalib/src/tests/net/async_resolver/async_resolver_test.cpp @@ -3,7 +3,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/net/async_resolver.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/util/sync.h> #include <atomic> using namespace vespalib; @@ -11,7 +10,7 @@ using namespace vespalib; struct ResultSetter : public AsyncResolver::ResultHandler { SocketAddress &addr; std::atomic<bool> done; - ResultSetter(SocketAddress &addr_out) : addr(addr_out), done(false) {} + ResultSetter(SocketAddress &addr_out) noexcept : addr(addr_out), done(false) {} void handle_result(SocketAddress result) override { addr = result; done = true; @@ -31,7 +30,7 @@ struct MyClock : public AsyncResolver::Clock { struct BlockingHostResolver : public AsyncResolver::HostResolver { CountDownLatch callers; Gate barrier; - BlockingHostResolver(size_t num_callers) + BlockingHostResolver(size_t num_callers) noexcept : callers(num_callers), barrier() {} vespalib::string ip_address(const vespalib::string &) override { callers.countDown(); diff --git a/vespalib/src/tests/portal/portal_test.cpp b/vespalib/src/tests/portal/portal_test.cpp index 0bd029c0c3a..07d3b9e1bfb 100644 --- a/vespalib/src/tests/portal/portal_test.cpp +++ b/vespalib/src/tests/portal/portal_test.cpp @@ -202,7 +202,7 @@ TEST("require that get requests dropped on the floor returns HTTP error") { vespalib::string path = "/test"; auto portal = Portal::create(null_crypto(), 0); auto expect = make_expected_error(500, "Internal Server Error"); - MyGetHandler handler([](Portal::GetRequest){}); + MyGetHandler handler([](Portal::GetRequest) noexcept {}); auto bound = portal->bind(path, handler); auto result = fetch(portal->listen_port(), null_crypto(), path); EXPECT_EQUAL(result, expect); diff --git a/vespalib/src/tests/priority_queue/priority_queue_test.cpp b/vespalib/src/tests/priority_queue/priority_queue_test.cpp index 3457c2bde6b..af8dd853c6c 100644 --- a/vespalib/src/tests/priority_queue/priority_queue_test.cpp +++ b/vespalib/src/tests/priority_queue/priority_queue_test.cpp @@ -154,11 +154,11 @@ TEST("require that priority queue works with move-only objects") { struct MyItem { int value; int *ref; - MyItem(int v, int &r) : value(v), ref(&r) {} - MyItem(const MyItem &) = delete; - MyItem &operator=(const MyItem &) = delete; - MyItem(MyItem &&rhs) : value(rhs.value), ref(rhs.ref) { rhs.ref = 0; } - MyItem &operator=(MyItem &&rhs) { + MyItem(int v, int &r) noexcept : value(v), ref(&r) {} + MyItem(const MyItem &) noexcept = delete; + MyItem &operator=(const MyItem &) noexcept = delete; + MyItem(MyItem &&rhs) noexcept : value(rhs.value), ref(rhs.ref) { rhs.ref = 0; } + MyItem &operator=(MyItem &&rhs) noexcept { value = rhs.value; ref = rhs.ref; rhs.ref = 0; diff --git a/vespalib/src/tests/rwlock/.gitignore b/vespalib/src/tests/rwlock/.gitignore deleted file mode 100644 index 3ba74c11b14..00000000000 --- a/vespalib/src/tests/rwlock/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -rwlock_test -vespalib_rwlock_test_app diff --git a/vespalib/src/tests/rwlock/CMakeLists.txt b/vespalib/src/tests/rwlock/CMakeLists.txt deleted file mode 100644 index 2eda95a200c..00000000000 --- a/vespalib/src/tests/rwlock/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_rwlock_test_app TEST - SOURCES - rwlock_test.cpp - DEPENDS - vespalib -) -vespa_add_test(NAME vespalib_rwlock_test_app COMMAND vespalib_rwlock_test_app) diff --git a/vespalib/src/tests/rwlock/rwlock_test.cpp b/vespalib/src/tests/rwlock/rwlock_test.cpp deleted file mode 100644 index 66e73c3a5d9..00000000000 --- a/vespalib/src/tests/rwlock/rwlock_test.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/rwlock.h> - -using namespace vespalib; - -class RWLockTest : public TestApp -{ -public: - int Main() override; - static RWLockReader rbvReader(RWLock & lock) { RWLockReader r(lock); return r; } - static RWLockWriter rbvWriter(RWLock & lock) { RWLockWriter r(lock); return r; } -}; - - -int -RWLockTest::Main() -{ - TEST_INIT("rwlock_test"); - - RWLock lock; - EXPECT_TRUE(lock._givenLocks == 0); - { - EXPECT_TRUE(lock._givenLocks == 0); - RWLockReader r1(lock); - EXPECT_TRUE(lock._givenLocks == 1); - RWLockReader r2(lock); - EXPECT_TRUE(lock._givenLocks == 2); - RWLockReader r3(lock); - EXPECT_TRUE(lock._givenLocks == 3); - } - EXPECT_TRUE(lock._givenLocks == 0); - { - EXPECT_TRUE(lock._givenLocks == 0); - RWLockWriter w(lock); - EXPECT_TRUE(lock._givenLocks == -1); - } - EXPECT_TRUE(lock._givenLocks == 0); - { - RWLockReader rbv(rbvReader(lock)); - EXPECT_TRUE(lock._givenLocks == 1); - RWLockReader copy(rbv); - EXPECT_TRUE(lock._givenLocks == 1); - RWLockReader copy2(copy); - EXPECT_TRUE(lock._givenLocks == 1); - } - EXPECT_TRUE(lock._givenLocks == 0); - { - RWLock lock2; - RWLockReader copy(rbvReader(lock)); - EXPECT_TRUE(lock._givenLocks == 1); - RWLockReader copy2(rbvReader(lock2)); - EXPECT_TRUE(lock._givenLocks == 1); - EXPECT_TRUE(lock2._givenLocks == 1); - RWLockReader rbv(rbvReader(lock)); - EXPECT_TRUE(lock._givenLocks == 2); - copy=rbv; - EXPECT_TRUE(lock._givenLocks == 1); - copy2=copy; - EXPECT_TRUE(lock2._givenLocks == 0); - EXPECT_TRUE(lock._givenLocks == 1); - } - EXPECT_TRUE(lock._givenLocks == 0); - { - RWLockWriter rbv(rbvWriter(lock)); - EXPECT_TRUE(lock._givenLocks == -1); - RWLockWriter copy(rbv); - EXPECT_TRUE(lock._givenLocks == -1); - RWLockWriter copy2(copy); - EXPECT_TRUE(lock._givenLocks == -1); - } - EXPECT_TRUE(lock._givenLocks == 0); - - TEST_DONE(); -} - -TEST_APPHOOK(RWLockTest) diff --git a/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp b/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp index 5641d751f34..d3c562c583c 100644 --- a/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp +++ b/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp @@ -3,7 +3,6 @@ #include <vespa/vespalib/util/simple_thread_bundle.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/box.h> -#include <vespa/vespalib/util/sync.h> using namespace vespalib; using namespace vespalib::fixed_thread_bundle; diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index e86a9ad020a..2b190b6b2dc 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -15,14 +15,14 @@ namespace { struct Foo { int i; - Foo() : i(0) {} - Foo(int i_) : i(i_) {} + Foo() noexcept : i(0) {} + Foo(int i_) noexcept : i(i_) {} - bool operator==(const Foo& f) const + bool operator==(const Foo& f) const noexcept { return (i == f.i); } struct hash { - size_t operator() (const Foo& f) const { + size_t operator() (const Foo& f) const noexcept { return (f.i % 16); } }; @@ -33,7 +33,7 @@ namespace { TEST("test that hashValue gives expected response") { const char * s("abcdefghi"); - EXPECT_EQUAL(2878261200250560019ul, vespalib::hashValue(s)); + EXPECT_EQUAL(16203358805722239136ul, vespalib::hashValue(s)); EXPECT_EQUAL(vespalib::hashValue(s), vespalib::hashValue(s, strlen(s))); EXPECT_NOT_EQUAL(vespalib::hashValue(s), vespalib::hashValue(s, strlen(s)-1)); } @@ -332,10 +332,10 @@ TEST("test hash map with simple key and value type") class S { public: - explicit S(uint64_t l=0) : _a(l&0xfffffffful), _b(l>>32) { } + explicit S(uint64_t l=0) noexcept : _a(l&0xfffffffful), _b(l>>32) { } uint32_t hash() const { return _a; } uint32_t a() const { return _a; } - friend bool operator == (const S & a, const S & b) { return a._a == b._a && a._b == b._b; } + friend bool operator == (const S & a, const S & b) noexcept { return a._a == b._a && a._b == b._b; } private: uint32_t _a, _b; }; diff --git a/vespalib/src/tests/stllike/uniq_by_sort_map_hash.cpp b/vespalib/src/tests/stllike/uniq_by_sort_map_hash.cpp index 72a0c93c719..faee7680426 100644 --- a/vespalib/src/tests/stllike/uniq_by_sort_map_hash.cpp +++ b/vespalib/src/tests/stllike/uniq_by_sort_map_hash.cpp @@ -71,13 +71,13 @@ class Gid { public: struct hash { - size_t operator () (const Gid & g) const { return g.getGid()[0]; } + size_t operator () (const Gid & g) const noexcept { return g.getGid()[0]; } }; - Gid(unsigned int v=0) : _gid() { _gid[0] = _gid[1] = _gid[2] = v; } + Gid(unsigned int v=0) noexcept : _gid() { _gid[0] = _gid[1] = _gid[2] = v; } const unsigned int * getGid() const { return _gid; } - int cmp(const Gid & b) const { return memcmp(_gid, b._gid, sizeof(_gid)); } - bool operator < (const Gid & b) const { return cmp(b) < 0; } - bool operator == (const Gid & b) const { return cmp(b) == 0; } + int cmp(const Gid & b) const noexcept { return memcmp(_gid, b._gid, sizeof(_gid)); } + bool operator < (const Gid & b) const noexcept { return cmp(b) < 0; } + bool operator == (const Gid & b) const noexcept { return cmp(b) == 0; } private: unsigned int _gid[3]; }; @@ -85,15 +85,15 @@ private: class Slot { public: - Slot(unsigned int v=0) : _gid(v) { } + Slot(unsigned int v=0) noexcept : _gid(v) { } const Gid & getGid() const { return _gid; } - int cmp(const Slot & b) const { return _gid.cmp(b.getGid()); } + int cmp(const Slot & b) const noexcept { return _gid.cmp(b.getGid()); } private: Gid _gid; }; struct IndirectCmp : public std::binary_function<Slot*, Slot*, bool> { - bool operator()(const Slot* s1, const Slot* s2) { + bool operator() (const Slot* s1, const Slot* s2) noexcept { return s1->cmp(*s2) < 0; } }; diff --git a/vespalib/src/tests/sync/sync_test.cpp b/vespalib/src/tests/sync/sync_test.cpp index a17f9089877..3a2cf0ea7a0 100644 --- a/vespalib/src/tests/sync/sync_test.cpp +++ b/vespalib/src/tests/sync/sync_test.cpp @@ -3,6 +3,41 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/sync.h> +namespace vespalib { +class TryLock +{ +private: + friend class LockGuard; + friend class MonitorGuard; + + std::unique_lock<std::mutex> _guard; + std::condition_variable *_cond; + +public: + TryLock(const Monitor &mon) + : _guard(*mon._mutex, std::try_to_lock), + _cond(_guard ? mon._cond.get() : nullptr) + {} + ~TryLock() = default; + + TryLock(const TryLock &) = delete; + TryLock &operator=(const TryLock &) = delete; + + /** + * @brief Check whether this object holds a lock + * + * @return true if this object holds a lock + **/ + bool hasLock() const { return static_cast<bool>(_guard); } + void unlock() { + if (_guard) { + _guard.unlock(); + _cond = nullptr; + } + } +}; + +} using namespace vespalib; #define CHECK_LOCKED(m) { TryLock tl(m); EXPECT_TRUE(!tl.hasLock()); } @@ -11,19 +46,18 @@ using namespace vespalib; class Test : public TestApp { private: - Lock _lock; Monitor _monitor; - LockGuard lockLock() { return LockGuard(_lock); } LockGuard lockMonitor() { return LockGuard(_monitor); } MonitorGuard obtainMonitor() { return MonitorGuard(_monitor); } public: - ~Test(); + ~Test() override; void testCountDownLatch(); int Main() override; }; -Test::~Test() {} +Test::~Test() = default; + void Test::testCountDownLatch() { { @@ -70,21 +104,6 @@ int Test::Main() { TEST_INIT("sync_test"); - { - Lock lock; - { - CHECK_UNLOCKED(lock); - LockGuard guard(lock); - CHECK_LOCKED(lock); - } - CHECK_UNLOCKED(lock); - { - LockGuard guard(lock); - CHECK_LOCKED(lock); - guard.unlock(); - CHECK_UNLOCKED(lock); - } - } // you can use a LockGuard to lock a Monitor { Monitor monitor; @@ -119,24 +138,9 @@ Test::Main() CHECK_UNLOCKED(monitor); } } - // copy/assign is nop, but legal - { - Lock a; - Lock b(a); - b = a; - } - { - Monitor a; - Monitor b(a); - b = a; - } + // you can lock const objects - { - const Lock lock; - CHECK_UNLOCKED(lock); - LockGuard guard(lock); - CHECK_LOCKED(lock); - } + { const Monitor lock; CHECK_UNLOCKED(lock); @@ -149,100 +153,9 @@ Test::Main() MonitorGuard guard(monitor); CHECK_LOCKED(monitor); } - // TryLock hands the lock over to a LockGuard/MonitorGuard - { - Lock lock; - CHECK_UNLOCKED(lock); - TryLock a(lock); - CHECK_LOCKED(lock); - if (a.hasLock()) { - LockGuard guard(std::move(a)); - CHECK_LOCKED(lock); - } - CHECK_UNLOCKED(lock); - } - { - Monitor mon; - CHECK_UNLOCKED(mon); - TryLock a(mon); - CHECK_LOCKED(mon); - if (a.hasLock()) { - LockGuard guard(std::move(a)); - CHECK_LOCKED(mon); - } - CHECK_UNLOCKED(mon); - } - { - Monitor mon; - CHECK_UNLOCKED(mon); - TryLock a(mon); - CHECK_LOCKED(mon); - if (a.hasLock()) { - MonitorGuard guard(std::move(a)); - CHECK_LOCKED(mon); - } - CHECK_UNLOCKED(mon); - } - { - Lock lock; - - CHECK_UNLOCKED(lock); - TryLock a(lock); - CHECK_LOCKED(lock); - TryLock b(lock); - CHECK_LOCKED(lock); - - EXPECT_TRUE(a.hasLock()); - EXPECT_TRUE(!b.hasLock()); - { - CHECK_LOCKED(lock); - EXPECT_TRUE(a.hasLock()); - LockGuard guard(std::move(a)); - EXPECT_TRUE(!a.hasLock()); - CHECK_LOCKED(lock); - } - CHECK_UNLOCKED(lock); - } - // TryLock will unlock when exiting scope if lock was not passed on - { - Lock lock; - Monitor mon; - CHECK_UNLOCKED(lock); - CHECK_UNLOCKED(mon); - { - TryLock a(lock); - EXPECT_TRUE(a.hasLock()); - TryLock b(mon); - EXPECT_TRUE(b.hasLock()); - CHECK_LOCKED(lock); - CHECK_LOCKED(mon); - } - CHECK_UNLOCKED(lock); - CHECK_UNLOCKED(mon); - } - // TryLock explicitt unlock of lock - { - Lock lock; - TryLock tl(lock); - EXPECT_TRUE(tl.hasLock()); - tl.unlock(); - EXPECT_FALSE(tl.hasLock()); - tl.unlock(); - EXPECT_FALSE(tl.hasLock()); - } - // TryLock explicitt unlock of monitor - { - Monitor lock; - TryLock tl(lock); - EXPECT_TRUE(tl.hasLock()); - tl.unlock(); - EXPECT_FALSE(tl.hasLock()); - tl.unlock(); - EXPECT_FALSE(tl.hasLock()); - } // LockGuard/MonitorGuard have destructive move { - Lock lock; + Monitor lock; CHECK_UNLOCKED(lock); LockGuard a(lock); CHECK_LOCKED(lock); @@ -267,19 +180,13 @@ Test::Main() } // Destructive copy also works for return value handover { - CHECK_UNLOCKED(_lock); CHECK_UNLOCKED(_monitor); { - CHECK_UNLOCKED(_lock); CHECK_UNLOCKED(_monitor); - LockGuard a(lockLock()); - CHECK_LOCKED(_lock); CHECK_UNLOCKED(_monitor); LockGuard b = lockMonitor(); // copy, not assign - CHECK_LOCKED(_lock); CHECK_LOCKED(_monitor); } - CHECK_UNLOCKED(_lock); CHECK_UNLOCKED(_monitor); } { @@ -293,8 +200,8 @@ Test::Main() } // Test that guards can be matched to locks/monitors { - Lock lock1; - Lock lock2; + Monitor lock1; + Monitor lock2; LockGuard lockGuard1(lock1); LockGuard lockGuard2(lock2); EXPECT_TRUE(lockGuard1.locks(lock1)); diff --git a/vespalib/src/tests/time_tracer/time_tracer_test.cpp b/vespalib/src/tests/time_tracer/time_tracer_test.cpp index d7e2e0d579a..6d55e867ba5 100644 --- a/vespalib/src/tests/time_tracer/time_tracer_test.cpp +++ b/vespalib/src/tests/time_tracer/time_tracer_test.cpp @@ -67,8 +67,8 @@ TEST("require that records are extracted inversely ordered by end time per threa } TEST("benchmark time sampling") { - double min_stamp_us = 1000000.0 * BenchmarkTimer::benchmark([](){ (void) TimeTracer::now(); }, 1.0); - double min_sample_us = 1000000.0 * BenchmarkTimer::benchmark([](){ TT_Sample my_sample(my_tag); }, 1.0); + double min_stamp_us = 1000000.0 * BenchmarkTimer::benchmark([]() noexcept { (void) TimeTracer::now(); }, 1.0); + double min_sample_us = 1000000.0 * BenchmarkTimer::benchmark([]() noexcept { TT_Sample my_sample(my_tag); }, 1.0); fprintf(stderr, "min timestamp time: %g us\n", min_stamp_us); fprintf(stderr, "min sample time: %g us\n", min_sample_us); fprintf(stderr, "estimated non-clock overhead: %g us\n", (min_sample_us - (min_stamp_us * 2.0))); diff --git a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp index a61f9fb5bca..2c1463e506a 100644 --- a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp +++ b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp @@ -15,7 +15,7 @@ struct WorkContext { uint64_t _generation; - WorkContext() + WorkContext() noexcept : _generation(0) { } diff --git a/vespalib/src/vespa/vespalib/btree/btree_key_data.h b/vespalib/src/vespa/vespalib/btree/btree_key_data.h index dc44faf00a9..5d4929694d5 100644 --- a/vespalib/src/vespa/vespalib/btree/btree_key_data.h +++ b/vespalib/src/vespa/vespalib/btree/btree_key_data.h @@ -28,12 +28,12 @@ public: KeyT _key; DataT _data; - BTreeKeyData() + BTreeKeyData() noexcept : _key(), _data() {} - BTreeKeyData(const KeyT &key, const DataT &data) + BTreeKeyData(const KeyT &key, const DataT &data) noexcept : _key(key), _data(data) {} @@ -60,9 +60,9 @@ public: KeyT _key; - BTreeKeyData() : _key() {} + BTreeKeyData() noexcept : _key() {} - BTreeKeyData(const KeyT &key, const BTreeNoLeafData &) + BTreeKeyData(const KeyT &key, const BTreeNoLeafData &) noexcept : _key(key) { } diff --git a/vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 04c4b1e225b..6b98226e50e 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -11,7 +11,7 @@ size_t padbefore(size_t alignment, const char *buf) { } } -DataBuffer::DataBuffer(size_t len, size_t alignment, const Alloc & initial) +DataBuffer::DataBuffer(size_t len, size_t alignment, const Alloc & initial) noexcept : _alignment(alignment), _externalBuf(nullptr), _bufstart(nullptr), diff --git a/vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h index 7c4cd63a7b1..93da2b92379 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.h +++ b/vespalib/src/vespa/vespalib/data/databuffer.h @@ -44,8 +44,8 @@ public: typedef std::unique_ptr<DataBuffer> UP; DataBuffer(const DataBuffer &) = delete; DataBuffer &operator=(const DataBuffer &) = delete; - DataBuffer(DataBuffer &&) = default; - DataBuffer &operator=(DataBuffer &&) = default; + DataBuffer(DataBuffer &&) noexcept = default; + DataBuffer &operator=(DataBuffer &&) noexcept = default; /** * Construct a databuffer. @@ -53,7 +53,7 @@ public: * @param len the initial size of the buffer. * @param alignment required memory alignment for data start **/ - DataBuffer(size_t len = 1024, size_t alignment = 1, const Alloc & initial = Alloc::alloc(0)); + DataBuffer(size_t len = 1024, size_t alignment = 1, const Alloc & initial = Alloc::alloc(0)) noexcept; /** * Construct a databuffer using externally allocated memory. Note @@ -63,7 +63,7 @@ public: * @param buf pointer to preallocated memory * @param len length of preallocated memory **/ - DataBuffer(void *buf, size_t len) : + DataBuffer(void *buf, size_t len) noexcept : _alignment(1), _externalBuf(static_cast<char *>(buf)), _bufstart(_externalBuf), @@ -73,7 +73,7 @@ public: _buffer(Alloc::alloc(0)) { } - DataBuffer(const void *buf, size_t len) : + DataBuffer(const void *buf, size_t len) noexcept : _alignment(1), _externalBuf(static_cast<char *>(const_cast<void *>(buf))), _bufstart(_externalBuf), diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp index df1b68cff12..1f8c72a5a64 100644 --- a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp +++ b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp @@ -5,8 +5,9 @@ namespace vespalib { using alloc::Alloc; +using LockGuard = std::lock_guard<std::mutex>; -MemoryDataStore::MemoryDataStore(Alloc && initialAlloc, Lock * lock) : +MemoryDataStore::MemoryDataStore(Alloc && initialAlloc, std::mutex * lock) : _buffers(), _writePos(0), _lock(lock) diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.h b/vespalib/src/vespa/vespalib/data/memorydatastore.h index 2d02bfb107c..fa6113279da 100644 --- a/vespalib/src/vespa/vespalib/data/memorydatastore.h +++ b/vespalib/src/vespa/vespalib/data/memorydatastore.h @@ -3,8 +3,8 @@ #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/array.h> -#include <vespa/vespalib/util/sync.h> #include <vector> +#include <mutex> namespace vespalib { @@ -24,7 +24,7 @@ public: private: void * _data; }; - MemoryDataStore(alloc::Alloc && initialAlloc=alloc::Alloc::alloc(256), Lock * lock=nullptr); + MemoryDataStore(alloc::Alloc && initialAlloc=alloc::Alloc::alloc(256), std::mutex * lock=nullptr); MemoryDataStore(const MemoryDataStore &) = delete; MemoryDataStore & operator = (const MemoryDataStore &) = delete; ~MemoryDataStore(); @@ -41,7 +41,7 @@ public: private: std::vector<alloc::Alloc> _buffers; size_t _writePos; - Lock * _lock; + std::mutex * _lock; }; class VariableSizeVector diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp index d2f953b38c8..080e3b8da48 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/locale/c.h> #include <cmath> #include <sstream> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".vespalib.data.slime.json_format"); diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index 6523cd1dac0..3ee608799a6 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -86,7 +86,7 @@ public: ~Slime(); - Slime(Slime &&rhs) : + Slime(Slime &&rhs) noexcept : _names(std::move(rhs._names)), _stash(std::move(rhs._stash)), _root(std::move(rhs._root)) diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.h b/vespalib/src/vespa/vespalib/datastore/entryref.h index 51a4e0699fc..4a5123ee1b3 100644 --- a/vespalib/src/vespa/vespalib/datastore/entryref.h +++ b/vespalib/src/vespa/vespalib/datastore/entryref.h @@ -13,13 +13,13 @@ class EntryRef { protected: uint32_t _ref; public: - EntryRef() : _ref(0u) { } - explicit EntryRef(uint32_t ref_) : _ref(ref_) { } - uint32_t ref() const { return _ref; } - bool valid() const { return _ref != 0u; } - bool operator==(const EntryRef &rhs) const { return _ref == rhs._ref; } - bool operator!=(const EntryRef &rhs) const { return _ref != rhs._ref; } - bool operator <(const EntryRef &rhs) const { return _ref < rhs._ref; } + EntryRef() noexcept : _ref(0u) { } + explicit EntryRef(uint32_t ref_) noexcept : _ref(ref_) { } + uint32_t ref() const noexcept { return _ref; } + bool valid() const noexcept { return _ref != 0u; } + bool operator==(const EntryRef &rhs) const noexcept { return _ref == rhs._ref; } + bool operator!=(const EntryRef &rhs) const noexcept { return _ref != rhs._ref; } + bool operator <(const EntryRef &rhs) const noexcept { return _ref < rhs._ref; } }; /** @@ -29,9 +29,9 @@ public: template <uint32_t OffsetBits, uint32_t BufferBits = 32u - OffsetBits> class EntryRefT : public EntryRef { public: - EntryRefT() : EntryRef() {} - EntryRefT(size_t offset_, uint32_t bufferId_); - EntryRefT(const EntryRef & ref_) : EntryRef(ref_.ref()) {} + EntryRefT() noexcept : EntryRef() {} + EntryRefT(size_t offset_, uint32_t bufferId_) noexcept; + EntryRefT(const EntryRef & ref_) noexcept : EntryRef(ref_.ref()) {} size_t offset() const { return _ref & (offsetSize() - 1); } uint32_t bufferId() const { return _ref >> OffsetBits; } static size_t offsetSize() { return 1ul << OffsetBits; } @@ -55,10 +55,10 @@ private: typedef EntryRefT<OffsetBits> ParentType; static const uint32_t PadConstant = ((1 << OffsetAlign) - 1); public: - AlignedEntryRefT() : ParentType() {} - AlignedEntryRefT(size_t offset_, uint32_t bufferId_) : + AlignedEntryRefT() noexcept : ParentType() {} + AlignedEntryRefT(size_t offset_, uint32_t bufferId_) noexcept : ParentType(align(offset_) >> OffsetAlign, bufferId_) {} - AlignedEntryRefT(const EntryRef & ref_) : ParentType(ref_) {} + AlignedEntryRefT(const EntryRef & ref_) noexcept : ParentType(ref_) {} size_t offset() const { return ParentType::offset() << OffsetAlign; } static size_t offsetSize() { return ParentType::offsetSize() << OffsetAlign; } static size_t align(size_t val) { return val + pad(val); } diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.hpp b/vespalib/src/vespa/vespalib/datastore/entryref.hpp index 56ebe62dfab..34b8d5355e3 100644 --- a/vespalib/src/vespa/vespalib/datastore/entryref.hpp +++ b/vespalib/src/vespa/vespalib/datastore/entryref.hpp @@ -8,7 +8,7 @@ namespace vespalib::datastore { template <uint32_t OffsetBits, uint32_t BufferBits> -EntryRefT<OffsetBits, BufferBits>::EntryRefT(size_t offset_, uint32_t bufferId_) : +EntryRefT<OffsetBits, BufferBits>::EntryRefT(size_t offset_, uint32_t bufferId_) noexcept : EntryRef((bufferId_ << OffsetBits) + offset_) { ASSERT_ONCE_OR_LOG(offset_ < offsetSize(), "EntryRefT.offset_overflow", 10000); diff --git a/vespalib/src/vespa/vespalib/net/socket_address.h b/vespalib/src/vespa/vespalib/net/socket_address.h index 2b75eae8948..543af44a56e 100644 --- a/vespalib/src/vespa/vespalib/net/socket_address.h +++ b/vespalib/src/vespa/vespalib/net/socket_address.h @@ -48,7 +48,7 @@ public: vespalib::string name() const; vespalib::string spec() const; SocketHandle connect(const std::function<bool(SocketHandle&)> &tweak) const; - SocketHandle connect() const { return connect([](SocketHandle&){ return true; }); } + SocketHandle connect() const { return connect([](SocketHandle&) noexcept { return true; }); } SocketHandle connect_async() const { return connect([](SocketHandle &handle){ return handle.set_blocking(false); }); } diff --git a/vespalib/src/vespa/vespalib/portal/portal.h b/vespalib/src/vespa/vespalib/portal/portal.h index 1c5b9f88a5f..5ac2d85a6e3 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.h +++ b/vespalib/src/vespa/vespalib/portal/portal.h @@ -53,7 +53,7 @@ public: GetRequest(const GetRequest &rhs) = delete; GetRequest &operator=(const GetRequest &rhs) = delete; GetRequest &operator=(GetRequest &&rhs) = delete; - GetRequest(GetRequest &&rhs) : _conn(rhs._conn) { + GetRequest(GetRequest &&rhs) noexcept : _conn(rhs._conn) { rhs._conn = nullptr; } bool active() const { return (_conn != nullptr); } diff --git a/vespalib/src/vespa/vespalib/portal/reactor.h b/vespalib/src/vespa/vespalib/portal/reactor.h index a7961c3c943..408f358d31b 100644 --- a/vespalib/src/vespa/vespalib/portal/reactor.h +++ b/vespalib/src/vespa/vespalib/portal/reactor.h @@ -60,7 +60,7 @@ private: public: Reactor(std::function<int()> tick); - Reactor() : Reactor([](){ return -1; }) {} + Reactor() : Reactor([]() noexcept { return -1; }) {} ~Reactor(); Token::UP attach(EventHandler &handler, int fd, bool read, bool write); }; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp index 5f4fee06c4a..84882d763d6 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp @@ -6,7 +6,7 @@ namespace vespalib { size_t -hashValue(const char *str) +hashValue(const char *str) noexcept { return hashValue(str, strlen(str)); } @@ -14,15 +14,15 @@ hashValue(const char *str) /** * @brief Calculate hash value. * - * The hash function XXH64 from xxhash library. + * The hash function XXH3_64bits from xxhash library. * @param buf input buffer * @param sz input buffer size * @return hash value of input **/ size_t -hashValue(const void * buf, size_t sz) +hashValue(const void * buf, size_t sz) noexcept { - return XXH64(buf, sz, 0); + return XXH3_64bits(buf, sz); } } diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.h b/vespalib/src/vespa/vespalib/stllike/hash_fun.h index c5f239f2b4e..089e8b14039 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.h @@ -8,71 +8,71 @@ namespace vespalib { template<typename K> struct hash { // specializations operate as functor for known key types - size_t operator() (const K & v) const { + size_t operator() (const K & v) const noexcept(noexcept(v.hash())) { return v.hash(); } }; template<> struct hash<char> { - size_t operator() (char arg) const { return arg; } + size_t operator() (char arg) const noexcept { return arg; } }; template<> struct hash<signed char> { - size_t operator() (signed char arg) const { return arg; } + size_t operator() (signed char arg) const noexcept { return arg; } }; template<> struct hash<short> { - size_t operator() (short arg) const { return arg; } + size_t operator() (short arg) const noexcept { return arg; } }; template<> struct hash<int> { - size_t operator() (int arg) const { return arg; } + size_t operator() (int arg) const noexcept { return arg; } }; template<> struct hash<long> { - size_t operator() (long arg) const { return arg; } + size_t operator() (long arg) const noexcept { return arg; } }; template<> struct hash<long long> { - size_t operator() (long long arg) const { return arg; } + size_t operator() (long long arg) const noexcept { return arg; } }; template<> struct hash<unsigned char> { - size_t operator() (unsigned char arg) const { return arg; } + size_t operator() (unsigned char arg) const noexcept { return arg; } }; template<> struct hash<unsigned short> { - size_t operator() (unsigned short arg) const { return arg; } + size_t operator() (unsigned short arg) const noexcept { return arg; } }; template<> struct hash<unsigned int> { - size_t operator() (unsigned int arg) const { return arg; } + size_t operator() (unsigned int arg) const noexcept { return arg; } }; template<> struct hash<unsigned long> { - size_t operator() (unsigned long arg) const { return arg; } + size_t operator() (unsigned long arg) const noexcept { return arg; } }; template<> struct hash<unsigned long long> { - size_t operator() (unsigned long long arg) const { return arg; } + size_t operator() (unsigned long long arg) const noexcept { return arg; } }; template<> struct hash<float> { union U { float f; uint32_t i; }; - size_t operator() (float arg) const { U t; t.f = arg; return t.i; } + size_t operator() (float arg) const noexcept { U t; t.f = arg; return t.i; } }; template<> struct hash<double> { union U { double f; uint64_t i; }; - size_t operator() (double arg) const { U t; t.f = arg; return t.i; } + size_t operator() (double arg) const noexcept { U t; t.f = arg; return t.i; } }; template<typename T> struct hash<T *> { - size_t operator() (const T * arg) const { return size_t(arg); } + size_t operator() (const T * arg) const noexcept { return size_t(arg); } }; template<typename T> struct hash<const T *> { - size_t operator() (const T * arg) const { return size_t(arg); } + size_t operator() (const T * arg) const noexcept { return size_t(arg); } }; // reuse old string hash function -size_t hashValue(const char *str); -size_t hashValue(const void *str, size_t sz); +size_t hashValue(const char *str) noexcept; +size_t hashValue(const void *str, size_t sz) noexcept; struct hash_strings { - size_t operator() (const vespalib::string & arg) const { return hashValue(arg.c_str()); } - size_t operator() (vespalib::stringref arg) const { return hashValue(arg.data(), arg.size()); } - size_t operator() (const char * arg) const { return hashValue(arg); } - size_t operator() (const std::string& arg) const { return hashValue(arg.c_str()); } + size_t operator() (const vespalib::string & arg) const noexcept { return hashValue(arg.c_str()); } + size_t operator() (vespalib::stringref arg) const noexcept { return hashValue(arg.data(), arg.size()); } + size_t operator() (const char * arg) const noexcept { return hashValue(arg); } + size_t operator() (const std::string& arg) const noexcept { return hashValue(arg.c_str()); } }; template<> struct hash<const char *> : hash_strings { }; @@ -81,11 +81,11 @@ template<> struct hash<vespalib::string> : hash_strings {}; template<> struct hash<std::string> : hash_strings {}; template<typename V> struct size { - size_t operator() (const V & arg) const { return arg.size(); } + size_t operator() (const V & arg) const noexcept { return arg.size(); } }; template<typename V> struct zero { - size_t operator() (const V & ) const { return 0; } + size_t operator() (const V & ) const noexcept { return 0; } }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/stllike/string.h b/vespalib/src/vespa/vespalib/stllike/string.h index 7d4ed06c411..f1207c1d9b8 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.h +++ b/vespalib/src/vespa/vespalib/stllike/string.h @@ -180,7 +180,7 @@ public: small_string(const char * s) noexcept : _buf(_stack), _sz(s ? strlen(s) : 0) { init(s); } small_string(const void * s, size_type sz) noexcept : _buf(_stack), _sz(sz) { init(s); } small_string(stringref s) noexcept : _buf(_stack), _sz(s.size()) { init(s.data()); } - small_string(const std::string & s) : _buf(_stack), _sz(s.size()) { init(s.data()); } + small_string(const std::string & s) noexcept : _buf(_stack), _sz(s.size()) { init(s.data()); } small_string(small_string && rhs) noexcept : _sz(rhs.size()), _bufferSize(rhs._bufferSize) { diff --git a/vespalib/src/vespa/vespalib/testkit/test_hook.h b/vespalib/src/vespa/vespalib/testkit/test_hook.h index 28419c0b31b..706a1c9f741 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_hook.h +++ b/vespalib/src/vespa/vespalib/testkit/test_hook.h @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/barrier.h> #include <string> #include <vector> +#include <cassert> #include "test_master.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.cpp b/vespalib/src/vespa/vespalib/testkit/test_master.cpp index 789d40d478d..60e51ac3a91 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.cpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.cpp @@ -3,6 +3,7 @@ #include "test_master.h" #include <vespa/vespalib/util/barrier.h> #include <cstring> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".vespalib.testkit.test_master"); @@ -33,10 +34,10 @@ __thread TestMaster::ThreadState *TestMaster::_threadState = 0; //----------------------------------------------------------------------------- -TestMaster::TraceItem::~TraceItem() { } +TestMaster::TraceItem::~TraceItem() = default; TestMaster::ThreadState & -TestMaster::threadState(const vespalib::LockGuard &) +TestMaster::threadState(const lock_guard &) { if (_threadState == 0) { std::ostringstream threadName; @@ -54,7 +55,7 @@ TestMaster::threadState() return *_threadState; } { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); return threadState(guard); } } @@ -62,7 +63,7 @@ TestMaster::threadState() //----------------------------------------------------------------------------- void -TestMaster::checkFailed(const vespalib::LockGuard &guard, +TestMaster::checkFailed(const lock_guard &guard, const char *file, uint32_t line, const char *str) { ThreadState &thread = threadState(guard); @@ -80,7 +81,7 @@ TestMaster::checkFailed(const vespalib::LockGuard &guard, } void -TestMaster::printDiff(const vespalib::LockGuard &guard, +TestMaster::printDiff(const lock_guard &guard, const std::string &text, const std::string &file, uint32_t line, const std::string &lhs, const std::string &rhs) { @@ -105,7 +106,7 @@ TestMaster::printDiff(const vespalib::LockGuard &guard, } void -TestMaster::handleFailure(const vespalib::LockGuard &guard, bool fatal) +TestMaster::handleFailure(const lock_guard &guard, bool fatal) { ThreadState &thread = threadState(guard); if (fatal) { @@ -119,7 +120,7 @@ TestMaster::handleFailure(const vespalib::LockGuard &guard, bool fatal) } void -TestMaster::closeDebugFiles(const vespalib::LockGuard &) +TestMaster::closeDebugFiles(const lock_guard &) { if (_state.lhsFile != NULL) { fclose(_state.lhsFile); @@ -132,7 +133,7 @@ TestMaster::closeDebugFiles(const vespalib::LockGuard &) } void -TestMaster::importThreads(const vespalib::LockGuard &) +TestMaster::importThreads(const lock_guard &) { size_t importCnt = 0; for (size_t i = 0; i < _threadStorage.size(); ++i) { @@ -148,7 +149,7 @@ TestMaster::importThreads(const vespalib::LockGuard &) } bool -TestMaster::reportConclusion(const vespalib::LockGuard &) +TestMaster::reportConclusion(const lock_guard &) { bool ok = (_state.failCnt == 0); fprintf(stderr, "%s: info: summary --- %zu check(s) passed --- %zu check(s) failed\n", @@ -174,7 +175,7 @@ TestMaster::TestMaster() void TestMaster::init(const char *name) { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); _name = skip_path(name); fprintf(stderr, "%s: info: running test suite '%s'\n", _name.c_str(), _name.c_str()); } @@ -182,7 +183,7 @@ TestMaster::init(const char *name) std::string TestMaster::getName() { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); return _name; } @@ -225,7 +226,7 @@ TestMaster::setThreadIgnore(bool ignore) size_t revertCnt = (thread.failCnt - thread.preIgnoreFailCnt); thread.failCnt = thread.preIgnoreFailCnt; if (revertCnt > 0) { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); assert(_state.failCnt >= revertCnt); _state.failCnt -= revertCnt; } @@ -271,7 +272,7 @@ TestMaster::getThreadFailCnt() TestMaster::Progress TestMaster::getProgress() { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); return Progress(_state.passCnt, _state.failCnt); } @@ -279,7 +280,7 @@ void TestMaster::openDebugFiles(const std::string &lhsFile, const std::string &rhsFile) { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); closeDebugFiles(guard); _state.lhsFile = fopen(lhsFile.c_str(), "w"); _state.rhsFile = fopen(rhsFile.c_str(), "w"); @@ -317,7 +318,7 @@ TestMaster::check(bool rc, const char *file, uint32_t line, return true; } { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); checkFailed(guard, file, line, str); handleFailure(guard, fatal); } @@ -329,7 +330,7 @@ TestMaster::flush(const char *file, uint32_t line) { ThreadState &thread = threadState(); if (thread.passCnt > 0) { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); _state.passCnt += thread.passCnt; fprintf(stderr, "%s: info: flushed %zu passed check(s) from thread '%s' (%s:%d)\n", _name.c_str(), thread.passCnt, thread.name.c_str(), skip_path(file), line); @@ -348,7 +349,7 @@ TestMaster::trace(const char *file, uint32_t line) bool TestMaster::discardFailedChecks(size_t failCnt) { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); ThreadState &thread = threadState(guard); if (failCnt == _state.failCnt) { fprintf(stderr, "%s: info: discarding %zu failed check(s)\n", _name.c_str(), _state.failCnt); @@ -366,7 +367,7 @@ TestMaster::discardFailedChecks(size_t failCnt) bool TestMaster::fini() { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); closeDebugFiles(guard); importThreads(guard); return reportConclusion(guard); diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.h b/vespalib/src/vespa/vespalib/testkit/test_master.h index a9dc5ebb3a2..c20982b994d 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.h +++ b/vespalib/src/vespa/vespalib/testkit/test_master.h @@ -2,10 +2,10 @@ #pragma once -#include <vespa/vespalib/util/sync.h> #include <string> #include <vector> #include <memory> +#include <mutex> namespace vespalib { @@ -69,24 +69,25 @@ private: }; private: - vespalib::Lock _lock; + std::mutex _lock; std::string _name; std::string _path_prefix; SharedState _state; std::vector<std::unique_ptr<ThreadState> > _threadStorage; + using lock_guard = std::lock_guard<std::mutex>; private: - ThreadState &threadState(const vespalib::LockGuard &); + ThreadState &threadState(const lock_guard &); ThreadState &threadState(); - void checkFailed(const vespalib::LockGuard &, + void checkFailed(const lock_guard &, const char *file, uint32_t line, const char *str); - void printDiff(const vespalib::LockGuard &, + void printDiff(const lock_guard &, const std::string &text, const std::string &file, uint32_t line, const std::string &lhs, const std::string &rhs); - void handleFailure(const vespalib::LockGuard &, bool do_abort); - void closeDebugFiles(const vespalib::LockGuard &); - void importThreads(const vespalib::LockGuard &); - bool reportConclusion(const vespalib::LockGuard &); + void handleFailure(const lock_guard &, bool do_abort); + void closeDebugFiles(const lock_guard &); + void importThreads(const lock_guard &); + bool reportConclusion(const lock_guard &); private: TestMaster(); diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.hpp b/vespalib/src/vespa/vespalib/testkit/test_master.hpp index f165458c9aa..245c128b788 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.hpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.hpp @@ -33,7 +33,7 @@ TestMaster::compare(const char *file, uint32_t line, lhs << a; rhs << b; { - vespalib::LockGuard guard(_lock); + lock_guard guard(_lock); checkFailed(guard, file, line, str.c_str()); printDiff(guard, str, file, line, lhs.str(), rhs.str()); handleFailure(guard, fatal); diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 0973653861f..ea2189fc3a8 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -40,7 +40,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT reusable_set_pool.cpp runnable.cpp runnable_pair.cpp - rwlock.cpp sequence.cpp sha1.cpp sig_catch.cpp @@ -50,6 +49,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT stash.cpp string_hash.cpp stringfmt.cpp + sync.cpp thread.cpp thread_bundle.cpp threadstackexecutor.cpp diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index 29285932f63..745fb50db03 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -4,10 +4,11 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/backtrace.h> -#include <vespa/vespalib/util/sync.h> #include <map> #include <atomic> #include <unordered_map> +#include <cassert> +#include <mutex> #include <vespa/fastos/file.h> #include <unistd.h> @@ -24,7 +25,7 @@ int _G_HugeFlags = 0; const size_t _G_pageSize = getpagesize(); size_t _G_MMapLogLimit = std::numeric_limits<size_t>::max(); size_t _G_MMapNoCoreLimit = std::numeric_limits<size_t>::max(); -Lock _G_lock; +std::mutex _G_lock; std::atomic<size_t> _G_mmapCount(0); size_t @@ -361,7 +362,7 @@ MMapAllocator::salloc(size_t sz, void * wantedAddress) } #endif if (sz >= _G_MMapLogLimit) { - LockGuard guard(_G_lock); + std::lock_guard guard(_G_lock); _G_HugeMappings[buf] = MMapInfo(mmapId, sz, stackTrace); LOG(info, "%ld mappings of accumulated size %ld", _G_HugeMappings.size(), sum(_G_HugeMappings)); } @@ -411,7 +412,7 @@ void MMapAllocator::sfree(PtrAndSize alloc) retval = munmap(alloc.first, alloc.second); assert(retval == 0); if (alloc.second >= _G_MMapLogLimit) { - LockGuard guard(_G_lock); + std::lock_guard guard(_G_lock); MMapInfo info = _G_HugeMappings[alloc.first]; assert(alloc.second == info._sz); _G_HugeMappings.erase(alloc.first); @@ -510,13 +511,13 @@ Alloc::allocMMap(size_t sz) } Alloc -Alloc::alloc() +Alloc::alloc() noexcept { return Alloc(&AutoAllocator::getDefault()); } Alloc -Alloc::alloc(size_t sz, size_t mmapLimit, size_t alignment) +Alloc::alloc(size_t sz, size_t mmapLimit, size_t alignment) noexcept { return Alloc(&AutoAllocator::getAllocator(mmapLimit, alignment), sz); } diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index 449cdde5fc7..cf7135736e5 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -81,18 +81,18 @@ public: } return *this; } - Alloc() : _alloc(nullptr, 0), _allocator(nullptr) { } + Alloc() noexcept : _alloc(nullptr, 0), _allocator(nullptr) { } ~Alloc() { if (_alloc.first != nullptr) { _allocator->free(_alloc); _alloc.first = nullptr; } } - void swap(Alloc & rhs) { + void swap(Alloc & rhs) noexcept { std::swap(_alloc, rhs._alloc); std::swap(_allocator, rhs._allocator); } - Alloc create(size_t sz) const { + Alloc create(size_t sz) const noexcept { return (sz == 0) ? Alloc(_allocator) : Alloc(_allocator, sz); } @@ -103,11 +103,11 @@ public: * Optional alignment is assumed to be <= system page size, since mmap * is always used when size is above limit. */ - static Alloc alloc(size_t sz, size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); - static Alloc alloc(); + static Alloc alloc(size_t sz, size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0) noexcept; + static Alloc alloc() noexcept; private: - Alloc(const MemoryAllocator * allocator, size_t sz) : _alloc(allocator->alloc(sz)), _allocator(allocator) { } - Alloc(const MemoryAllocator * allocator) : _alloc(nullptr, 0), _allocator(allocator) { } + Alloc(const MemoryAllocator * allocator, size_t sz) noexcept : _alloc(allocator->alloc(sz)), _allocator(allocator) { } + Alloc(const MemoryAllocator * allocator) noexcept : _alloc(nullptr, 0), _allocator(allocator) { } void clear() { _alloc.first = nullptr; _alloc.second = 0; diff --git a/vespalib/src/vespa/vespalib/util/arrayqueue.hpp b/vespalib/src/vespa/vespalib/util/arrayqueue.hpp index 17a8c02dbf5..9af446e7a0f 100644 --- a/vespalib/src/vespa/vespalib/util/arrayqueue.hpp +++ b/vespalib/src/vespa/vespalib/util/arrayqueue.hpp @@ -101,15 +101,16 @@ public: /** * Create an empty queue with an initial capacity of 0. **/ - ArrayQueue() : _data(0), _capacity(0), _used(0), _skew(0) {} + ArrayQueue() noexcept : _data(0), _capacity(0), _used(0), _skew(0) {} /** * Create an empty queue with the given initial capacity. * * @param cap initial capacity **/ - explicit ArrayQueue(uint32_t cap) : _data((T*)malloc(sizeof(T) * cap)), - _capacity(cap), _used(0), _skew(0) {} + explicit ArrayQueue(uint32_t cap) noexcept + : _data((T*)malloc(sizeof(T) * cap)), _capacity(cap), _used(0), _skew(0) + {} /** * Create a queue that is a copy of another queue. Now with funky @@ -119,8 +120,8 @@ public: * @param q the queue that should be copied **/ ArrayQueue(typename std::conditional<is_copyable<T>::value, void_tag, const ArrayQueue &>::type q) = delete; - ArrayQueue(typename std::conditional<is_copyable<T>::value, const ArrayQueue &, void_tag>::type q) : _data((T*)malloc(sizeof(T) * q._capacity)), - _capacity(q._capacity), _used(0), _skew(0) + ArrayQueue(typename std::conditional<is_copyable<T>::value, const ArrayQueue &, void_tag>::type q) + : _data((T*)malloc(sizeof(T) * q._capacity)), _capacity(q._capacity), _used(0), _skew(0) { try { q.copyInto(*this); @@ -136,7 +137,7 @@ public: * * @param q the queue that should be moved **/ - ArrayQueue(ArrayQueue &&q) : _data(0), _capacity(0), _used(0), _skew(0) + ArrayQueue(ArrayQueue &&q) noexcept : _data(0), _capacity(0), _used(0), _skew(0) { swap(q); } @@ -358,7 +359,7 @@ public: * * @param q the queue we want to swap state with **/ - void swap(ArrayQueue<T> &q) { + void swap(ArrayQueue<T> &q) noexcept { std::swap(_data, q._data); std::swap(_capacity, q._capacity); std::swap(_used, q._used); diff --git a/vespalib/src/vespa/vespalib/util/child_process.cpp b/vespalib/src/vespa/vespalib/util/child_process.cpp index ce0c2eb1779..b804d4ca87c 100644 --- a/vespalib/src/vespa/vespalib/util/child_process.cpp +++ b/vespalib/src/vespa/vespalib/util/child_process.cpp @@ -56,9 +56,6 @@ public: } // namespace child_process -using child_process::Timer; - -//----------------------------------------------------------------------------- void ChildProcess::Reader::OnReceiveData(const void *data, size_t length) @@ -88,7 +85,7 @@ ChildProcess::Reader::hasData() bool -ChildProcess::Reader::waitForData(Timer &timer, MonitorGuard &lock) +ChildProcess::Reader::waitForData(child_process::Timer &timer, MonitorGuard &lock) { // NB: caller has lock on _cond CounterGuard count(_waitCnt); @@ -131,7 +128,7 @@ ChildProcess::Reader::read(char *buf, uint32_t len, int msTimeout) if (eof()) { return 0; } - Timer timer(msTimeout); + child_process::Timer timer(msTimeout); MonitorGuard lock(_cond); waitForData(timer, lock); uint32_t bytes = 0; @@ -162,7 +159,7 @@ ChildProcess::Reader::readLine(std::string &line, int msTimeout) if (eof()) { return false; } - Timer timer(msTimeout); + child_process::Timer timer(msTimeout); MonitorGuard lock(_cond); while (waitForData(timer, lock)) { while (hasData()) { @@ -299,7 +296,7 @@ ChildProcess::run(const std::string &input, const char *cmd, std::string &output, int msTimeout) { ChildProcess proc(cmd); - Timer timer(msTimeout); + child_process::Timer timer(msTimeout); char buf[4096]; proc.write(input.data(), input.length()); proc.close(); // close stdin diff --git a/vespalib/src/vespa/vespalib/util/compressionconfig.h b/vespalib/src/vespa/vespalib/util/compressionconfig.h index c0010e8e05c..88563c181a1 100644 --- a/vespalib/src/vespa/vespalib/util/compressionconfig.h +++ b/vespalib/src/vespa/vespalib/util/compressionconfig.h @@ -20,15 +20,15 @@ struct CompressionConfig { ZSTD = 7 }; - CompressionConfig() + CompressionConfig() noexcept : type(NONE), compressionLevel(0), threshold(90), minSize(0) {} - CompressionConfig(Type t) + CompressionConfig(Type t) noexcept : type(t), compressionLevel(9), threshold(90), minSize(0) {} - CompressionConfig(Type t, uint8_t level, uint8_t minRes) + CompressionConfig(Type t, uint8_t level, uint8_t minRes) noexcept : type(t), compressionLevel(level), threshold(minRes), minSize(0) {} - CompressionConfig(Type t, uint8_t lvl, uint8_t minRes, size_t minSz) + CompressionConfig(Type t, uint8_t lvl, uint8_t minRes, size_t minSz) noexcept : type(t), compressionLevel(lvl), threshold(minRes), minSize(minSz) {} bool operator==(const CompressionConfig& o) const { diff --git a/vespalib/src/vespa/vespalib/util/count_down_latch.h b/vespalib/src/vespa/vespalib/util/count_down_latch.h index 66ef1e44cee..420be8f7ab2 100644 --- a/vespalib/src/vespa/vespalib/util/count_down_latch.h +++ b/vespalib/src/vespa/vespalib/util/count_down_latch.h @@ -37,7 +37,7 @@ public: * * @param cnt initial count **/ - CountDownLatch(uint32_t cnt) : _lock(), _cond(), _count(cnt) {} + CountDownLatch(uint32_t cnt) noexcept : _lock(), _cond(), _count(cnt) {} /** * Count down this latch. When the count reaches 0, all threads diff --git a/vespalib/src/vespa/vespalib/util/delegatelist.hpp b/vespalib/src/vespa/vespalib/util/delegatelist.hpp deleted file mode 100644 index 716474f66f7..00000000000 --- a/vespalib/src/vespa/vespalib/util/delegatelist.hpp +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "eventbarrier.hpp" -#include "sync.h" - -namespace vespalib { - -/** - * Data structure for robust event multi-casting in a multi-threaded - * environment. The state tracked by this class can be modeled as a - * set of bald object pointers; the delegates. All interaction with - * the delegates is done through a snapshot of the delegate list. The - * list may be modified at any time. Modifications will not be visible - * to already existing snapshots. A separate method may be used to - * wait for the destruction of all currently active snapshots. This - * synchronization will ensure visibility of any previous - * modifications to the delegate list. State snapshotting is - * implemented with reference counted immutable lists. Snapshot - * waiting is implemented using event barriers. - **/ -template <typename T> -class DelegateList -{ -private: - /** - * Simple class used to synchronize with the completion of an - * event barrier by coupling barrier completion to a gate. - **/ - struct Sync - { - Gate gate; - Sync() : gate() {} - void completeBarrier() { gate.countDown(); } - }; - - /** - * Inner structure used when keeping track of a set of delegates. - **/ - struct Node { - uint32_t refcnt; // the number of incoming pointers to this node. - T *delegate; // the delegate tracked by this node. - Node *next; // the next node in this list. - }; - - Lock _lock; // lock protecting access to this object - Node *_head; // head of current list of delegates - EventBarrier<Sync> _barrier; // object used to resolve event barriers - Node *_freeNodes; // a list of recyclable internal nodes - int _activeNodes; // number of internal nodes currently in use - ArrayQueue<T*> _stack; // explicit stack for cheap 'recursion' - - /** - * Allocate a new node and initialize it with the given data - * members. Nodes are recycled internally by this object to reduce - * overhead. - * - * @return the new node - * @param delegate the delegate for the new node - * @param next the next pointer for the new node - **/ - Node *allocNode(T *delegate, Node *next) { - Node *node = _freeNodes; - if (node != 0) { - _freeNodes = node->next; - } else { - node = new Node(); - } - node->refcnt = 1; - node->delegate = delegate; - node->next = next; - ++_activeNodes; - return node; - } - - /** - * Copy a list of nodes. This will increase the reference count of - * the list. - * - * @return the copy of the list - * @param list the list to copy - **/ - Node *copyList(Node *list) { - if (list != 0) { - ++list->refcnt; - } - return list; - } - - /** - * Free a list of nodes. This will decrease the reference count of - * the list. Any nodes no longer in use will be put back on the - * internal free list. - * - * @return 0 - * @param list the list to free - **/ - Node *freeList(Node *list) { - while (list != 0 && --list->refcnt == 0) { - Node *node = list; - list = node->next; - node->next = _freeNodes; - _freeNodes = node; - --_activeNodes; - } - return 0; - } - - DelegateList(const DelegateList &); - DelegateList &operator=(const DelegateList &); - -public: - /** - * A snapshot of a delegate list. The only way to access the - * delegates kept by a delegate list is to create a snapshot of - * it. The snapshot lets the user traverse the list of delegates, - * accessing each of them in turn. The existence of a snapshot is - * used by the delegate list to identify that someone is observing - * the delegate list in a specific state. Snapshots should be - * created on the stack in a scope as close as possible to the - * code actually accessing the delegates. The delegate list itself - * may not be destructed until all snapshots of that list have - * been destructed. - **/ - class Snapshot - { - private: - DelegateList &_list; // the parent object - Node *_head; // head of the snapshotted list - Node *_node; // current position within snapshot - uint32_t _token; // token used for barrier resolving - - Snapshot(const Snapshot &); - Snapshot &operator=(const Snapshot &); - public: - /** - * Create a snapshot of the given delegate list. The snapshots - * current position will be set to the first delegate part of - * the snapshot. - * - * @param list the delegate list we are snapshotting - **/ - explicit Snapshot(DelegateList &list) : _list(list) { - LockGuard guard(list._lock); - _head = list.copyList(list._head); - _node = _head; - _token = list._barrier.startEvent(); - } - - /** - * Destructing a snapshot will tell the delegate list that we - * are no longer accessing the list in the state observed by - * the snapshot. - **/ - ~Snapshot() { - LockGuard guard(_list._lock); - _list.freeList(_head); - _list._barrier.completeEvent(_token); - } - - /** - * Check whether the current delegate is valid. A snapshot - * becomes invalid after the user has stepped through all - * delegates part of the snapshot using the 'next' method. - * - * @return true if the current delegate is valid - **/ - bool valid() const { - return (_node != 0); - } - - /** - * Step to the next delegate. This method may only be called - * if the 'valid' method returns true. - **/ - void next() { - _node = _node->next; - } - - /** - * Get the current delegate. This method may only be called if - * the 'valid' method returns true. - * - * @return current delegate - **/ - T *get() const { - return _node->delegate; - } - }; - - /** - * Create an initially empty delegate list. - **/ - DelegateList() - : _lock(), - _head(0), - _barrier(), - _freeNodes(0), - _activeNodes(0), - _stack() - { - } - - /** - * The destructor will clean up internal memory usage. The - * delegate list itself does not need to be empty when it is - * deleted. However, there may be no active snapshots of it and - * no-one may be waiting for snapshot destruction. - **/ - ~DelegateList() { - freeList(_head); - assert(_barrier.countBarriers() == 0); - assert(_barrier.countEvents() == 0); - assert(_activeNodes == 0); - while (_freeNodes != 0) { - Node *node = _freeNodes; - _freeNodes = node->next; - delete node; - } - } - - /** - * Add a delegate to this list. Adding a delegate that is already - * in the list will have no effect. - * - * @return this object, for chaining - * @param delegate the delegate to add - **/ - DelegateList &add(T *delegate) { - LockGuard guard(_lock); - Node *node = _head; - while (node != 0 && node->delegate != delegate) { - node = node->next; - } - if (node == 0) { - _head = allocNode(delegate, _head); - } - return *this; - } - - /** - * Remove a delegate from this list. - * - * @return this object, for chaining - * @param delegate the delegate to remove - **/ - DelegateList &remove(T *delegate) { - LockGuard guard(_lock); - _stack.clear(); - Node *node = _head; - while (node != 0 && node->delegate != delegate) { - _stack.push(node->delegate); - node = node->next; - } - if (node != 0) { // delegate found in list - node = copyList(node->next); - while (!_stack.empty()) { - try { - node = allocNode(_stack.back(), node); - } catch (...) { - freeList(node); - throw; - } - _stack.popBack(); - } - freeList(_head); - _head = node; - } - return *this; - } - - /** - * Remove all delegates currently in this list. - * - * @return this object, for chaining - **/ - DelegateList &clear() { - LockGuard guard(_lock); - _head = freeList(_head); - return *this; - } - - /** - * Wait for the destruction of all currently active snapshots of - * this list. This method will block until all relevant snapshots - * are destructed. The creation of new snapshots will not - * interfere with the completion of this method. This method is - * used to enforce visibility between threads; after this method - * returns, any modifications performed on the list before this - * method was invoked will be visible to all threads. - * - * @return this object, for chaining - **/ - DelegateList &waitSnapshots() { - Sync sync; - { - LockGuard guard(_lock); - if (!_barrier.startBarrier(sync)) { - return *this; - } - } - sync.gate.await(); - return *this; - } -}; - -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/util/gate.h b/vespalib/src/vespa/vespalib/util/gate.h index 7d913a7a039..5505a3676df 100644 --- a/vespalib/src/vespa/vespalib/util/gate.h +++ b/vespalib/src/vespa/vespalib/util/gate.h @@ -16,7 +16,7 @@ public: /** * Sets the initial count to 1. **/ - Gate() : CountDownLatch(1) {} + Gate() noexcept : CountDownLatch(1) {} }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/memory.h b/vespalib/src/vespa/vespalib/util/memory.h index f1c7c7820f1..2e0d631725a 100644 --- a/vespalib/src/vespa/vespalib/util/memory.h +++ b/vespalib/src/vespa/vespalib/util/memory.h @@ -200,7 +200,7 @@ public: * with default sz=0 you get an empty container * @param sz the number of bytes to allocate **/ - MallocPtr(const size_t sz=0) : _sz(sz), _p(_sz ? malloc(sz) : nullptr) { + MallocPtr(const size_t sz=0) noexcept : _sz(sz), _p(_sz ? malloc(sz) : nullptr) { if (_p == nullptr) { _sz = 0; } @@ -208,7 +208,7 @@ public: /** @brief destructor doing free() if needed */ ~MallocPtr() { cleanup(); } - MallocPtr(MallocPtr && rhs) : + MallocPtr(MallocPtr && rhs) noexcept : _sz(rhs.size()), _p(rhs._p) { rhs._sz = 0; @@ -221,7 +221,7 @@ public: * Does deep copy of contents (using memcpy). * @param rhs container to copy **/ - MallocPtr(const MallocPtr & rhs) + MallocPtr(const MallocPtr & rhs) noexcept : _sz(rhs.size()), _p(_sz ? malloc(_sz) : nullptr) { if (_p == nullptr) { @@ -238,14 +238,14 @@ public: * works like destruct + copy construct. * @param rhs container to copy **/ - MallocPtr & operator = (const MallocPtr & rhs) { + MallocPtr & operator = (const MallocPtr & rhs) noexcept { if (this != &rhs) { MallocPtr tmp(rhs); swap(tmp); } return *this; } - MallocPtr & operator = (MallocPtr && rhs) { + MallocPtr & operator = (MallocPtr && rhs) noexcept { if (this != &rhs) { cleanup(); _sz = rhs._sz; @@ -261,7 +261,7 @@ public: * * does not copy anything, just swaps pointers. **/ - void swap(MallocPtr & rhs) { + void swap(MallocPtr & rhs) noexcept { std::swap(_sz, rhs._sz); std::swap(_p, rhs._p); } @@ -307,7 +307,7 @@ public: } } private: - void cleanup() { + void cleanup() noexcept { if (_p) { free(_p); _p = nullptr; @@ -350,7 +350,7 @@ public: } /** @brief move constructor, takes over ownership */ - CloneablePtr(std::unique_ptr<T> &&rhs) + CloneablePtr(std::unique_ptr<T> &&rhs) noexcept : _p(rhs.release()) { } @@ -373,7 +373,7 @@ public: } /** @brief swap contents */ - void swap(CloneablePtr & rhs) { std::swap(_p, rhs._p); } + void swap(CloneablePtr & rhs) noexcept { std::swap(_p, rhs._p); } /** @brief value access */ const T * get() const { return _p; } diff --git a/vespalib/src/vespa/vespalib/util/ptrholder.h b/vespalib/src/vespa/vespalib/util/ptrholder.h index 1c06e5c53e6..be528ba25db 100644 --- a/vespalib/src/vespa/vespalib/util/ptrholder.h +++ b/vespalib/src/vespa/vespalib/util/ptrholder.h @@ -4,7 +4,7 @@ #include <algorithm> #include <memory> -#include <vespa/vespalib/util/sync.h> +#include <mutex> namespace vespalib { @@ -29,11 +29,11 @@ class PtrHolder private: std::shared_ptr<T> _current; std::shared_ptr<T> _next; - mutable Lock _lock; - - PtrHolder(const PtrHolder &); - PtrHolder &operator=(const PtrHolder &); + mutable std::mutex _lock; + using LockGuard = std::lock_guard<std::mutex>; public: + PtrHolder(const PtrHolder &) = delete; + PtrHolder &operator=(const PtrHolder &) = delete; /** * @brief Create an empty PtrHolder with both current and new * pointers set to 0 @@ -53,14 +53,14 @@ public: * * @return true if the current value is set (not 0) **/ - bool hasValue() const { return (_current.get() != nullptr); } + bool hasValue() const { return bool(_current); } /** * @brief Check if the new value is set (not 0) * * @return true if the new value is set (not 0) **/ - bool hasNewValue() const { return (_next.get() != nullptr); } + bool hasNewValue() const { return bool(_next); } /** * @brief Set a new value diff --git a/vespalib/src/vespa/vespalib/util/rwlock.cpp b/vespalib/src/vespa/vespalib/util/rwlock.cpp deleted file mode 100644 index d1e155851cf..00000000000 --- a/vespalib/src/vespa/vespalib/util/rwlock.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/util/rwlock.h> - -namespace vespalib { - -void RWLock::lockRead() { - MonitorGuard guard(_monitor); - CounterGuard waitCnt(_waitingReaders); - while (_givenLocks == -1 || _waitingWriters > 0) { - guard.wait(); - } - ++_givenLocks; -} - -void RWLock::unlockRead() { - MonitorGuard guard(_monitor); - assert(_givenLocks > 0); - if (--_givenLocks == 0 && _waitingWriters > 0) { - guard.broadcast(); - } -} - -void RWLock::lockWrite() { - MonitorGuard guard(_monitor); - CounterGuard waitCnt(_waitingWriters); - while (_givenLocks != 0) { - guard.wait(); - } - _givenLocks = -1; -} - -void RWLock::unlockWrite() { - MonitorGuard guard(_monitor); - assert(_givenLocks == -1); - _givenLocks = 0; - if (_waitingReaders > 0 || _waitingWriters > 0) { - guard.broadcast(); - } -} - -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/rwlock.h b/vespalib/src/vespa/vespalib/util/rwlock.h deleted file mode 100644 index 9b1f9b93289..00000000000 --- a/vespalib/src/vespa/vespalib/util/rwlock.h +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/util/guard.h> - -class RWLockTest; - -namespace vespalib { - -/** - * @brief An RWLock is a reader/writer lock. It can either be held by - * any number of readers or a single writer at any time. - * - * The RWLockReader and RWLockWriter classes are used to acquire and - * release reader and writer locks respectively. - * - * Writer locks have priority above reader locks to prevent - * starvation. - **/ -class RWLock -{ -private: - friend class ::RWLockTest; - friend class RWLockReader; - friend class RWLockWriter; - - int _givenLocks; - int _waitingReaders; - int _waitingWriters; - Monitor _monitor; - - void lockRead(); - void unlockRead(); - void lockWrite(); - void unlockWrite(); -public: - /** - * @brief Create a new RWLock - **/ - RWLock() - : _givenLocks(0), - _waitingReaders(0), - _waitingWriters(0), - _monitor() {} - /** - * @brief Create a new RWLock, ignoring the right hand side. - * - * It makes no sense to copy the state of an RWLock, but we want - * to allow copying objects that contain RWLock objects. - * - * @param rhs ignore this - **/ - RWLock(const RWLock &rhs) - : _givenLocks(0), - _waitingReaders(0), - _waitingWriters(0), - _monitor() { (void) rhs;} - /** - * @brief Assignment operator ignoring the right hand side. - * - * It makes no sense to assign the state of one RWLock to another, - * but we want to allow assigning objects that contain RWLock - * objects. - * - * @param rhs ignore this - **/ - RWLock &operator=(const RWLock &rhs) { - (void) rhs; - return *this; - } - - /** - * To get an instance of RWLockReader or RWLockWriter that isn't - * associated with a specific RWLock at initialization, you may - * construct them from this tag type. - **/ - struct InitiallyUnlockedGuard {}; -}; - -#ifndef IAM_DOXYGEN -class RWLockReaderHandover -{ -private: - friend class RWLockReader; - RWLock *_lock; - RWLockReaderHandover(const RWLockReaderHandover &); - RWLockReaderHandover &operator=(const RWLockReaderHandover &); - RWLockReaderHandover(RWLock *m) : _lock(m) {} -public: -}; - -class RWLockWriterHandover -{ -private: - friend class RWLockWriter; - RWLock *_lock; - RWLockWriterHandover(const RWLockWriterHandover &); - RWLockWriterHandover &operator=(const RWLockWriterHandover &); - RWLockWriterHandover(RWLock *m) : _lock(m) {} -public: -}; -#endif - - -/** - * @brief An RWLockReader holds a reader lock on an RWLock. - * - * The lock is acquired in the constructor and released in the - * destructor. - * - * RWLockReader has destructive copy (like unique_ptr). Assigning from - * or copying a RWLockReader has the semantic of transferring the lock - * from one object to the other. Note that assigning from or copying - * a RWLockReader that does not have a lock will result in an assert. - **/ -class RWLockReader -{ -private: - RWLock * _lock; - RWLock * stealLock() { - RWLock * ret(_lock); - assert(ret != NULL); - _lock = NULL; - return ret; - } - void cleanup() { if (_lock != NULL) { _lock->unlockRead(); } } -public: - - /** - * @brief Obtain reader lock. - * - * This will block until a reader lock can be acquired. - * - * @param lock the underlying RWLock object - **/ - RWLockReader(RWLock &lock) : _lock(&lock) { _lock->lockRead(); } - - /** - * @brief Construct initially unlocked guard. - * @param tag (unused) marker argument - **/ - RWLockReader(const RWLock::InitiallyUnlockedGuard &tag) : _lock(NULL) { (void)tag; } - - /** - * @brief Steal the lock from the given RWLockReader - * - * @param rhs steal the lock from this one - **/ - RWLockReader(RWLockReader &rhs) : _lock(rhs.stealLock()) {} - - /** - * @brief Steal the lock from the given RWLockReader - * - * @param rhs steal the lock from this one - **/ - RWLockReader &operator=(RWLockReader & rhs) { - if (this != & rhs) { - cleanup(); - _lock = rhs.stealLock(); - } - return *this; - } - - /** - * @brief Release the lock obtained in the constructor - **/ - ~RWLockReader() { cleanup(); } - -#ifndef IAM_DOXYGEN - RWLockReader(const RWLockReaderHandover &rhs) : _lock(rhs._lock) {} - operator RWLockReaderHandover() { return RWLockReaderHandover(stealLock()); } -#endif -}; - - -/** - * @brief An RWLockWriter holds a writer lock on an RWLock. - * - * The lock is acquired in the constructor and released in the - * destructor. - * - * RWLockWriter has destructive copy (like unique_ptr). Assigning from - * or copying a RWLockWriter has the semantic of transferring the lock - * from one object to the other, and assignment is similar. Note that - * assigning from or copying a RWLockWriter that does not have a lock - * will result in an assert. - **/ -class RWLockWriter -{ -private: - RWLock * _lock; - RWLock * stealLock() { - RWLock * ret(_lock); - assert(ret != NULL); - _lock = NULL; - return ret; - } - void cleanup() { if (_lock != NULL) { _lock->unlockWrite(); } } -public: - - /** - * @brief Obtain writer lock. - * - * This will block until a writer lock can be acquired. - * - * @param lock the underlying RWLock object - **/ - RWLockWriter(RWLock &lock) : _lock(&lock) { _lock->lockWrite(); } - - /** - * @brief Construct initially unlocked guard. - * @param tag (unused) marker argument - **/ - RWLockWriter(const RWLock::InitiallyUnlockedGuard &tag) : _lock(NULL) { (void)tag; } - - /** - * @brief Steal the lock from the given RWLockWriter - * - * @param rhs steal the lock from this one - **/ - RWLockWriter(RWLockWriter &rhs) : _lock(rhs.stealLock()) {} - - /** - * @brief Steal the lock from the given RWLockWriter - * - * @param rhs steal the lock from this one - **/ - RWLockWriter &operator=(RWLockWriter & rhs) { - if (this != & rhs) { - cleanup(); - _lock = rhs.stealLock(); - } - return *this; - } - - /** - * @brief Release the lock obtained in the constructor - **/ - ~RWLockWriter() { cleanup(); } - -#ifndef IAM_DOXYGEN - RWLockWriter(const RWLockWriterHandover &rhs) : _lock(rhs._lock) {} - operator RWLockWriterHandover() { return RWLockWriterHandover(stealLock()); } -#endif -}; - -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp index 01979c25164..e0d78c5d1b7 100644 --- a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp +++ b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.cpp @@ -2,6 +2,7 @@ #include "simple_thread_bundle.h" #include "exceptions.h" +#include <cassert> using namespace vespalib::fixed_thread_bundle; @@ -68,20 +69,20 @@ SimpleThreadBundle::UP SimpleThreadBundle::Pool::obtain() { { - LockGuard guard(_lock); + std::lock_guard guard(_lock); if (!_bundles.empty()) { SimpleThreadBundle::UP ret(_bundles.back()); _bundles.pop_back(); return ret; } } - return SimpleThreadBundle::UP(new SimpleThreadBundle(_bundleSize)); + return std::make_unique<SimpleThreadBundle>(_bundleSize); } void SimpleThreadBundle::Pool::release(SimpleThreadBundle::UP bundle) { - LockGuard guard(_lock); + std::lock_guard guard(_lock); _bundles.push_back(bundle.get()); bundle.release(); } diff --git a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h index 4fa73c1112f..135fc2d7562 100644 --- a/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h +++ b/vespalib/src/vespa/vespalib/util/simple_thread_bundle.h @@ -2,7 +2,6 @@ #pragma once -#include "sync.h" #include "count_down_latch.h" #include "thread.h" #include "runnable.h" @@ -48,7 +47,7 @@ struct Signal { bool valid; size_t generation; Monitor monitor; - Signal() : valid(true), generation(0), monitor() {} + Signal() noexcept : valid(true), generation(0), monitor() {} size_t wait(size_t &localGen) { MonitorGuard guard(monitor); while (localGen == generation) { @@ -94,8 +93,8 @@ public: class Pool { private: - Lock _lock; - size_t _bundleSize; + std::mutex _lock; + size_t _bundleSize; std::vector<SimpleThreadBundle*> _bundles; public: diff --git a/vespalib/src/vespa/vespalib/util/sync.cpp b/vespalib/src/vespa/vespalib/util/sync.cpp new file mode 100644 index 00000000000..1f7a5215c46 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/sync.cpp @@ -0,0 +1,99 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sync.h" +#include <cassert> + +namespace vespalib { + +Monitor::Monitor() noexcept + : _mutex(std::make_unique<std::mutex>()), + _cond(std::make_unique<std::condition_variable>()) +{} +Monitor::Monitor(Monitor &&rhs) noexcept = default; +Monitor::~Monitor() = default; + +LockGuard::LockGuard() : _guard() {} + +LockGuard::LockGuard(LockGuard &&rhs) noexcept : _guard(std::move(rhs._guard)) { } +LockGuard::LockGuard(const Monitor &lock) : _guard(*lock._mutex) { } + +LockGuard & +LockGuard::operator=(LockGuard &&rhs) noexcept{ + if (this != &rhs) { + _guard = std::move(rhs._guard); + } + return *this; +} + +void +LockGuard::unlock() { + if (_guard) { + _guard.unlock(); + } +} +LockGuard::~LockGuard() = default; + +bool +LockGuard::locks(const Monitor & lock) const { + return (_guard && _guard.mutex() == lock._mutex.get()); +} + +MonitorGuard::MonitorGuard() : _guard(), _cond(nullptr) {} +MonitorGuard::MonitorGuard(MonitorGuard &&rhs) noexcept + : _guard(std::move(rhs._guard)), + _cond(rhs._cond) +{ + rhs._cond = nullptr; +} +MonitorGuard::MonitorGuard(const Monitor &monitor) + : _guard(*monitor._mutex), + _cond(monitor._cond.get()) +{ } + +MonitorGuard & +MonitorGuard::operator=(MonitorGuard &&rhs) noexcept { + if (this != &rhs) { + _guard = std::move(rhs._guard); + _cond = rhs._cond; + rhs._cond = nullptr; + } + return *this; +} + +void +MonitorGuard::unlock() { + assert(_guard); + _guard.unlock(); + _cond = nullptr; +} +void +MonitorGuard::wait() { + _cond->wait(_guard); +} +bool +MonitorGuard::wait(int msTimeout) { + return wait(std::chrono::milliseconds(msTimeout)); +} +bool +MonitorGuard::wait(std::chrono::nanoseconds timeout) { + return _cond->wait_for(_guard, timeout) == std::cv_status::no_timeout; +} +void +MonitorGuard::signal() { + _cond->notify_one(); +} +void +MonitorGuard::broadcast() { + _cond->notify_all(); +} +void +MonitorGuard::unsafeSignalUnlock() { + _guard.unlock(); + _cond->notify_one(); + _cond = nullptr; +} + +MonitorGuard::~MonitorGuard() = default; + +} // namespace vespalib + diff --git a/vespalib/src/vespa/vespalib/util/sync.h b/vespalib/src/vespa/vespalib/util/sync.h index 8458bc19629..d66a53cf539 100644 --- a/vespalib/src/vespa/vespalib/util/sync.h +++ b/vespalib/src/vespa/vespalib/util/sync.h @@ -2,45 +2,13 @@ #pragma once -#include <cassert> +#include "time.h" #include <mutex> #include <condition_variable> -#include <chrono> namespace vespalib { /** - * @brief A Lock is a synchronization primitive used to ensure mutual - * exclusion. - * - * Use a LockGuard to hold a lock inside a scope. - * - * It is possible to obtain a lock on a const Lock object. - * - * @see TryLock - **/ -class Lock -{ -protected: - friend class LockGuard; - friend class TryLock; - - mutable std::mutex _mutex; -public: - /** - * @brief Create a new Lock. - * - * Creates a Lock that has mutex instrumentation disabled. - **/ - Lock() : _mutex() {} - Lock(const Lock &) : Lock() { } - Lock(Lock &&) : Lock() { } - Lock &operator=(const Lock &) { return *this; } - Lock &operator=(Lock &&) { return *this; } -}; - - -/** * @brief A Monitor is a synchronization primitive used to protect * data access and also facilitate signaling and waiting between * threads. @@ -52,133 +20,24 @@ public: * * @see TryLock **/ -class Monitor : public Lock +class Monitor { private: friend class LockGuard; friend class MonitorGuard; friend class TryLock; - mutable std::condition_variable _cond; + std::unique_ptr<std::mutex> _mutex; + std::unique_ptr<std::condition_variable> _cond; public: /** * @brief Create a new Monitor. * * Creates a Monitor that has mutex instrumentation disabled. **/ - Monitor() : Lock(), _cond() {} - Monitor(const Monitor &) : Monitor() { } - Monitor(Monitor &&) : Monitor() { } - Monitor &operator=(const Monitor &) { return *this; } - Monitor &operator=(Monitor &&) { return *this; } -}; - - -/** - * @brief A TryLock object is used to try to obtain the lock on a Lock - * or a Monitor without blocking. - * - * A TryLock will typically fail to obatin the lock if someone else - * already has it. In that case, the TryLock object has no further - * use. - * - * If the TryLock managed to acquire the lock, it can be passed over - * to a LockGuard or MonitorGuard object. If the lock is not passed - * on, the TryLock object will release it when it goes out of scope. - * - * Note that passing the lock obtained from a Lock to a MonitorGuard - * is illegal. Also note that if the TryLock fails to aquire the lock, - * it cannot be passed on. Trying to do so will result in an assert. - * - * copy/assignment of a TryLock is illegal. - * - * <pre> - * Example: - * - * Lock lock; - * TryLock tl(lock); - * if (tl.hasLock()) { - * LockGuard guard(tl) - * ... do stuff - * } // the lock is released as 'guard' goes out of scope - * </pre> - **/ -class TryLock -{ -private: - friend class LockGuard; - friend class MonitorGuard; - - std::unique_lock<std::mutex> _guard; - std::condition_variable *_cond; - - TryLock(const TryLock &) = delete; - TryLock &operator=(const TryLock &) = delete; - -public: - /** - * @brief Try to obtain the lock represented by the given Lock object - * - * @param lock the lock to obtain - **/ - TryLock(const Lock &lock) - : _guard(lock._mutex, std::try_to_lock), - _cond(nullptr) - { - } - - /** - * @brief Try to lock the given Monitor - * - * @param mon the monitor to lock - **/ - TryLock(const Monitor &mon) - : _guard(mon._mutex, std::try_to_lock), - _cond(_guard ? &mon._cond : nullptr) - { - } - - TryLock(TryLock &&rhs) - : _guard(std::move(rhs._guard)), - _cond(rhs._cond) - { - rhs._cond = nullptr; - } - - /** - * @brief Release the lock held by this object, if any - **/ - ~TryLock() = default; - - TryLock &operator=(TryLock &&rhs) { - if (this != &rhs) { - _guard = std::move(rhs._guard); - _cond = rhs._cond; - rhs._cond = nullptr; - } - return *this; - } - - /** - * @brief Check whether this object holds a lock - * - * @return true if this object holds a lock - **/ - bool hasLock() { return static_cast<bool>(_guard); } - /** - * @brief Release the lock held by this object. - * - * No methods may be invoked after invoking unlock (except the - * destructor). Note that this method should only be used if you - * need to release the lock before the object is destructed, as - * the destructor will release the lock. - **/ - void unlock() { - if (_guard) { - _guard.unlock(); - _cond = nullptr; - } - } + Monitor() noexcept; + Monitor(Monitor && rhs) noexcept; + ~Monitor(); }; @@ -198,19 +57,20 @@ class LockGuard { private: std::unique_lock<std::mutex> _guard; - LockGuard &operator=(const LockGuard &) = delete; public: /** * @brief A noop guard without any mutex. **/ - LockGuard() : _guard() {} + LockGuard(); LockGuard(const LockGuard &rhs) = delete; + LockGuard &operator=(const LockGuard &) = delete; + /** * @brief Steal the lock from the given LockGuard * * @param rhs steal the lock from this one **/ - LockGuard(LockGuard &&rhs) : _guard(std::move(rhs._guard)) { } + LockGuard(LockGuard &&rhs) noexcept; /** * @brief Obtain the lock represented by the given Lock object. * @@ -218,28 +78,9 @@ public: * * @param lock take it **/ - LockGuard(const Lock &lock) : _guard(lock._mutex) { } - - /** - * @brief Create a LockGuard from a TryLock. - * - * The TryLock may have been created from either a Lock or a - * Monitor, but it must have managed to acquire the lock. The lock - * will be handed over from the TryLock to the new object. - * - * @param tlock take the lock from this one - **/ - LockGuard(TryLock &&tlock) : _guard(std::move(tlock._guard)) - { - tlock._cond = nullptr; - } + LockGuard(const Monitor &lock); - LockGuard &operator=(LockGuard &&rhs) { - if (this != &rhs) { - _guard = std::move(rhs._guard); - } - return *this; - } + LockGuard &operator=(LockGuard &&rhs) noexcept; /** * @brief Release the lock held by this object. @@ -249,24 +90,18 @@ public: * need to release the lock before the object is destructed, as * the destructor will release the lock. **/ - void unlock() { - if (_guard) { - _guard.unlock(); - } - } + void unlock(); /** * @brief Release the lock held by this object if unlock has not * been called. **/ - ~LockGuard() = default; + ~LockGuard(); /** * Allow code to match guard with lock. This allows functions to take a * guard ref as input, ensuring that the caller have grabbed a lock. */ - bool locks(const Lock& lock) const { - return (_guard && _guard.mutex() == &lock._mutex); - } + bool locks(const Monitor& lock) const; }; @@ -294,19 +129,14 @@ public: /** * @brief A noop guard without any condition. **/ - MonitorGuard() : _guard(), _cond(nullptr) {} + MonitorGuard(); MonitorGuard(const MonitorGuard &rhs) = delete; /** * @brief Steal the lock from the given MonitorGuard * * @param rhs steal the lock from this one **/ - MonitorGuard(MonitorGuard &&rhs) - : _guard(std::move(rhs._guard)), - _cond(rhs._cond) - { - rhs._cond = nullptr; - } + MonitorGuard(MonitorGuard &&rhs) noexcept; /** * @brief Obtain the lock on the given Monitor object. * @@ -314,39 +144,9 @@ public: * * @param monitor take the lock on it **/ - MonitorGuard(const Monitor &monitor) - : _guard(monitor._mutex), - _cond(&monitor._cond) - { - } - /** - * @brief Create a MonitorGuard from a TryLock. - * - * The TryLock must have been created from a Monitor, and it must - * have managed to acquire the lock. The lock will be handed over - * from the TryLock to the new object. - * - * @param tlock take the lock from this one - **/ - MonitorGuard(TryLock &&tlock) - : _guard(), - _cond(nullptr) - { - if (tlock._guard && tlock._cond != nullptr) { - _guard = std::move(tlock._guard); - _cond = tlock._cond; - tlock._cond = nullptr; - } - } + MonitorGuard(const Monitor &monitor); - MonitorGuard &operator=(MonitorGuard &&rhs) { - if (this != &rhs) { - _guard = std::move(rhs._guard); - _cond = rhs._cond; - rhs._cond = nullptr; - } - return *this; - } + MonitorGuard &operator=(MonitorGuard &&rhs) noexcept; /** @@ -357,17 +157,11 @@ public: * need to release the lock before this object is destructed, as * the destructor will release the lock. **/ - void unlock() { - assert(_guard); - _guard.unlock(); - _cond = nullptr; - } + void unlock(); /** * @brief Wait for a signal on the underlying Monitor. **/ - void wait() { - _cond->wait(_guard); - } + void wait(); /** * @brief Wait for a signal on the underlying Monitor with the * given timeout. @@ -375,25 +169,17 @@ public: * @param msTimeout timeout in milliseconds * @return true if a signal was received, false if the wait timed out. **/ - bool wait(int msTimeout) { - return wait(std::chrono::milliseconds(msTimeout)); - } - bool wait(std::chrono::nanoseconds timeout) { - return _cond->wait_for(_guard, timeout) == std::cv_status::no_timeout; - } + bool wait(int msTimeout); + bool wait(duration timeout); /** * @brief Send a signal to a single waiter on the underlying * Monitor. **/ - void signal() { - _cond->notify_one(); - } + void signal(); /** * @brief Send a signal to all waiters on the underlying Monitor. **/ - void broadcast() { - _cond->notify_all(); - } + void broadcast(); /** * @brief Send a signal to a single waiter on the underlying * Monitor, but unlock the monitor right before doing so. @@ -402,24 +188,20 @@ public: * synchronization to ensure that the underlying Monitor object * will live long enough to be signaled. **/ - void unsafeSignalUnlock() { - _guard.unlock(); - _cond->notify_one(); - _cond = nullptr; - } + void unsafeSignalUnlock(); /** * @brief Release the lock held by this object if unlock has not * been called. **/ - ~MonitorGuard() = default; + ~MonitorGuard(); /** * Allow code to match guard with lock. This allows functions to take a * guard ref as input, ensuring that the caller have grabbed a lock. */ bool monitors(const Monitor& m) const { - return (_cond != nullptr && _cond == &m._cond); + return (_cond != nullptr && _cond == m._cond.get()); } }; diff --git a/vespalib/src/vespa/vespalib/util/thread.cpp b/vespalib/src/vespa/vespalib/util/thread.cpp index 4eb436458a2..c595a76b8fe 100644 --- a/vespalib/src/vespa/vespalib/util/thread.cpp +++ b/vespalib/src/vespa/vespalib/util/thread.cpp @@ -2,6 +2,7 @@ #include "thread.h" #include <thread> +#include <cassert> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp index 2f09abf4846..931f727ad6c 100644 --- a/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp +++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp @@ -2,9 +2,7 @@ #include "zstdcompressor.h" #include <vespa/vespalib/util/alloc.h> -#include <vespa/vespalib/util/sync.h> #include <zstd.h> -#include <vector> #include <cassert> using vespalib::alloc::Alloc; diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp index 8307954faae..db238279fa3 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -116,7 +116,7 @@ DocsumTools::obtainFieldNames(const FastS_VsmsummaryHandle &cfg) void VSMAdapter::configure(const VSMConfigSnapshot & snapshot) { - vespalib::LockGuard guard(_lock); + std::lock_guard guard(_lock); LOG(debug, "(re-)configure VSM (docsum tools)"); std::shared_ptr<SummaryConfig> summary(snapshot.getConfig<SummaryConfig>().release()); @@ -141,7 +141,7 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) } // init keyword extractor - std::unique_ptr<KeywordExtractor> kwExtractor(new KeywordExtractor(NULL)); + auto kwExtractor = std::make_unique<KeywordExtractor>(nullptr); kwExtractor->AddLegalIndexSpec(_highlightindexes.c_str()); vespalib::string spec = kwExtractor->GetLegalIndexSpec(); LOG(debug, "index highlight spec: '%s'", spec.c_str()); diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/vsm/src/vespa/vsm/vsm/vsm-adapter.h index 31e472713de..f6895dd71a6 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.h @@ -127,7 +127,7 @@ private: vespalib::PtrHolder<DocsumTools> _docsumTools; std::unique_ptr<JuniperProperties> _juniperProps; - vespalib::Lock _lock; + std::mutex _lock; VSMAdapter(const VSMAdapter &); VSMAdapter &operator=(const VSMAdapter &); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java index 1478b36d331..9cf490bf8c6 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java @@ -21,19 +21,21 @@ public class LockAttempt { private final String lockPath; private final Instant callAcquireInstant; private final Duration timeout; + private final boolean reentry; private final LockMetrics lockMetrics; private final List<LockAttempt> nestedLockAttempts = new ArrayList<>(); private final LatencyStats.ActiveInterval activeAcquireInterval; // Only accessed by mutating thread: - private LatencyStats.ActiveInterval activeLockedInterval = null; + private Optional<LatencyStats.ActiveInterval> activeLockedInterval = Optional.empty(); private volatile Optional<Instant> lockAcquiredInstant = Optional.empty(); private volatile Optional<Instant> terminalStateInstant = Optional.empty(); private volatile Optional<String> stackTrace = Optional.empty(); public static LockAttempt invokingAcquire(ThreadLockStats threadLockStats, String lockPath, - Duration timeout, LockMetrics lockMetrics) { - return new LockAttempt(threadLockStats, lockPath, timeout, Instant.now(), lockMetrics); + Duration timeout, LockMetrics lockMetrics, + boolean reentry) { + return new LockAttempt(threadLockStats, lockPath, timeout, Instant.now(), lockMetrics, reentry); } public enum LockState { @@ -50,13 +52,14 @@ public class LockAttempt { private volatile LockState lockState = LockState.ACQUIRING; private LockAttempt(ThreadLockStats threadLockStats, String lockPath, Duration timeout, - Instant callAcquireInstant, LockMetrics lockMetrics) { + Instant callAcquireInstant, LockMetrics lockMetrics, boolean reentry) { this.threadLockStats = threadLockStats; this.lockPath = lockPath; this.callAcquireInstant = callAcquireInstant; this.timeout = timeout; this.lockMetrics = lockMetrics; - this.activeAcquireInterval = lockMetrics.acquireInvoked(); + this.reentry = reentry; + this.activeAcquireInterval = lockMetrics.acquireInvoked(reentry); } public String getThreadName() { return threadLockStats.getThreadName(); } @@ -101,22 +104,22 @@ public class LockAttempt { void acquireFailed() { setTerminalState(LockState.ACQUIRE_FAILED); - lockMetrics.acquireFailed(activeAcquireInterval); + lockMetrics.acquireFailed(reentry, activeAcquireInterval); } void timedOut() { setTerminalState(LockState.TIMED_OUT); - lockMetrics.acquireTimedOut(activeAcquireInterval); + lockMetrics.acquireTimedOut(reentry, activeAcquireInterval); } void lockAcquired() { lockState = LockState.ACQUIRED; lockAcquiredInstant = Optional.of(Instant.now()); - activeLockedInterval = lockMetrics.lockAcquired(activeAcquireInterval); + activeLockedInterval = Optional.of(lockMetrics.lockAcquired(reentry, activeAcquireInterval)); } void preRelease() { - lockMetrics.preRelease(activeLockedInterval); + lockMetrics.preRelease(reentry, activeLockedInterval.orElseThrow()); } void postRelease() { @@ -125,7 +128,7 @@ public class LockAttempt { void releaseFailed() { setTerminalState(LockState.RELEASED_WITH_ERROR); - lockMetrics.releaseFailed(); + lockMetrics.releaseFailed(reentry); } void setTerminalState(LockState terminalState) { setTerminalState(terminalState, Instant.now()); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockMetrics.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockMetrics.java index 18ab70d42da..36758354171 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockMetrics.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockMetrics.java @@ -17,6 +17,7 @@ public class LockMetrics { private final AtomicInteger acquireSucceededCount = new AtomicInteger(0); private final AtomicInteger releaseCount = new AtomicInteger(0); private final AtomicInteger releaseFailedCount = new AtomicInteger(0); + private final AtomicInteger reentryCount = new AtomicInteger(0); private final AtomicInteger cumulativeAcquireCount = new AtomicInteger(0); private final AtomicInteger cumulativeAcquireFailedCount = new AtomicInteger(0); @@ -24,43 +25,55 @@ public class LockMetrics { private final AtomicInteger cumulativeAcquireSucceededCount = new AtomicInteger(0); private final AtomicInteger cumulativeReleaseCount = new AtomicInteger(0); private final AtomicInteger cumulativeReleaseFailedCount = new AtomicInteger(0); + private final AtomicInteger cumulativeReentryCount = new AtomicInteger(0); private final LatencyStats acquireStats = new LatencyStats(); private final LatencyStats lockedStats = new LatencyStats(); /** Returns a Runnable that must be invoked when the acquire() finishes. */ - ActiveInterval acquireInvoked() { + ActiveInterval acquireInvoked(boolean reentry) { + if (reentry) { + reentryCount.incrementAndGet(); + cumulativeReentryCount.incrementAndGet(); + return () -> { }; + } + acquireCount.incrementAndGet(); cumulativeAcquireCount.incrementAndGet(); return acquireStats.startNewInterval(); } - void acquireFailed(ActiveInterval acquireInterval) { + void acquireFailed(boolean reentry, ActiveInterval acquireInterval) { acquireInterval.close(); + if (reentry) return; acquireFailedCount.incrementAndGet(); cumulativeAcquireFailedCount.incrementAndGet(); } - void acquireTimedOut(ActiveInterval acquireInterval) { + void acquireTimedOut(boolean reentry, ActiveInterval acquireInterval) { acquireInterval.close(); + if (reentry) return; acquireTimedOutCount.incrementAndGet(); cumulativeAcquireTimedOutCount.incrementAndGet(); } - ActiveInterval lockAcquired(ActiveInterval acquireInterval) { + ActiveInterval lockAcquired(boolean reentry, ActiveInterval acquireInterval) { acquireInterval.close(); + if (reentry) return () -> {}; acquireSucceededCount.incrementAndGet(); cumulativeAcquireSucceededCount.incrementAndGet(); return lockedStats.startNewInterval(); } - void preRelease(ActiveInterval lockedInterval) { + void preRelease(boolean reentry, ActiveInterval lockedInterval) { lockedInterval.close(); + if (reentry) return; releaseCount.incrementAndGet(); cumulativeReleaseCount.incrementAndGet(); } - void releaseFailed() { + void releaseFailed(boolean reentry) { + if (reentry) return; releaseFailedCount.incrementAndGet(); cumulativeReleaseFailedCount.incrementAndGet(); } @@ -71,6 +84,7 @@ public class LockMetrics { public int getAndResetAcquireSucceededCount() { return acquireSucceededCount.getAndSet(0); } public int getAndResetReleaseCount() { return releaseCount.getAndSet(0); } public int getAndResetReleaseFailedCount() { return releaseFailedCount.getAndSet(0); } + public int getAndResetReentryCount() { return reentryCount.getAndSet(0); } public int getCumulativeAcquireCount() { return cumulativeAcquireCount.get(); } public int getCumulativeAcquireFailedCount() { return cumulativeAcquireFailedCount.get(); } @@ -78,6 +92,7 @@ public class LockMetrics { public int getCumulativeAcquireSucceededCount() { return cumulativeAcquireSucceededCount.get(); } public int getCumulativeReleaseCount() { return cumulativeReleaseCount.get(); } public int getCumulativeReleaseFailedCount() { return cumulativeReleaseFailedCount.get(); } + public int getCumulativeReentryCount() { return cumulativeReentryCount.get(); } public LatencyMetrics getAcquireLatencyMetrics() { return acquireStats.getLatencyMetrics(); } public LatencyMetrics getLockedLatencyMetrics() { return lockedStats.getLatencyMetrics(); } @@ -86,20 +101,22 @@ public class LockMetrics { public LatencyMetrics getAndResetLockedLatencyMetrics() { return lockedStats.getLatencyMetricsAndStartNewPeriod(); } // For tests - void setAcquireCount(int count) { acquireCount.set(count); } - void setAcquireFailedCount(int count) { acquireFailedCount.set(count); } - void setAcquireTimedOutCount(int count) { acquireTimedOutCount.set(count); } - void setAcquireSucceededCount(int count) { acquireSucceededCount.set(count); } - void setReleaseCount(int count) { releaseCount.set(count); } - void setReleaseFailedCount(int count) { releaseFailedCount.set(count); } + LockMetrics setAcquireCount(int count) { acquireCount.set(count); return this; } + LockMetrics setAcquireFailedCount(int count) { acquireFailedCount.set(count); return this; } + LockMetrics setAcquireTimedOutCount(int count) { acquireTimedOutCount.set(count); return this; } + LockMetrics setAcquireSucceededCount(int count) { acquireSucceededCount.set(count); return this; } + LockMetrics setReleaseCount(int count) { releaseCount.set(count); return this; } + LockMetrics setReleaseFailedCount(int count) { releaseFailedCount.set(count); return this; } + LockMetrics setReentryCount(int count) { reentryCount.set(count); return this; } // For tests - void setCumulativeAcquireCount(int count) { cumulativeAcquireCount.set(count); } - void setCumulativeAcquireFailedCount(int count) { cumulativeAcquireFailedCount.set(count); } - void setCumulativeAcquireTimedOutCount(int count) { cumulativeAcquireTimedOutCount.set(count); } - void setCumulativeAcquireSucceededCount(int count) { cumulativeAcquireSucceededCount.set(count); } - void setCumulativeReleaseCount(int count) { cumulativeReleaseCount.set(count); } - void setCumulativeReleaseFailedCount(int count) { cumulativeReleaseFailedCount.set(count); } + LockMetrics setCumulativeAcquireCount(int count) { cumulativeAcquireCount.set(count); return this; } + LockMetrics setCumulativeAcquireFailedCount(int count) { cumulativeAcquireFailedCount.set(count); return this; } + LockMetrics setCumulativeAcquireTimedOutCount(int count) { cumulativeAcquireTimedOutCount.set(count); return this; } + LockMetrics setCumulativeAcquireSucceededCount(int count) { cumulativeAcquireSucceededCount.set(count); return this; } + LockMetrics setCumulativeReleaseCount(int count) { cumulativeReleaseCount.set(count); return this; } + LockMetrics setCumulativeReleaseFailedCount(int count) { cumulativeReleaseFailedCount.set(count); return this; } + LockMetrics setCumulativeReentryCount(int count) { cumulativeReentryCount.set(count); return this; } @Override public String toString() { @@ -110,12 +127,14 @@ public class LockMetrics { ", acquireSucceededCount=" + acquireSucceededCount + ", releaseCount=" + releaseCount + ", releaseFailedCount=" + releaseFailedCount + + ", reentryCount=" + reentryCount + ", cumulativeAcquireCount=" + cumulativeAcquireCount + ", cumulativeAcquireFailedCount=" + cumulativeAcquireFailedCount + ", cumulativeAcquireTimedOutCount=" + cumulativeAcquireTimedOutCount + ", cumulativeAcquireSucceededCount=" + cumulativeAcquireSucceededCount + ", cumulativeReleaseCount=" + cumulativeReleaseCount + ", cumulativeReleaseFailedCount=" + cumulativeReleaseFailedCount + + ", cumulativeReentryCount=" + cumulativeReentryCount + ", acquireStats=" + acquireStats + ", lockedStats=" + lockedStats + '}'; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java index 964aa83c52f..393fac5e3db 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java @@ -64,7 +64,10 @@ public class ThreadLockStats { /** Mutable method (see class doc) */ public void invokingAcquire(String lockPath, Duration timeout) { - LockAttempt lockAttempt = LockAttempt.invokingAcquire(this, lockPath, timeout, getGlobalLockMetrics(lockPath)); + boolean reentry = lockAttemptsStack.stream().anyMatch(lockAttempt -> lockAttempt.getLockPath().equals(lockPath)); + + LockAttempt lockAttempt = LockAttempt.invokingAcquire(this, lockPath, timeout, + getGlobalLockMetrics(lockPath), reentry); LockAttempt lastLockAttempt = lockAttemptsStack.peekLast(); if (lastLockAttempt == null) { diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java index 252a90f8bb4..e54e340deb5 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java @@ -54,7 +54,7 @@ public class LockAttemptSamplesTest { private boolean maybeSample(String lockPath, int secondsDuration) { LockAttempt lockAttempt = LockAttempt.invokingAcquire(threadLockStats, lockPath, - Duration.ofSeconds(1), new LockMetrics()); + Duration.ofSeconds(1), new LockMetrics(), false); Instant instant = lockAttempt.getTimeAcquiredWasInvoked().plus(Duration.ofSeconds(secondsDuration)); lockAttempt.setTerminalState(LockAttempt.LockState.RELEASED, instant); return samples.maybeSample(lockAttempt); diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java index d673655c798..f440c2cfad8 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -26,8 +26,10 @@ import static org.mockito.Mockito.when; public class LockTest { private final InterProcessLock mutex = mock(InterProcessLock.class); private final String lockPath = "/lock/path"; + private final String lock2Path = "/lock2/path"; private final Duration acquireTimeout = Duration.ofSeconds(10); private final Lock lock = new Lock(lockPath, mutex); + private final Lock lock2 = new Lock(lock2Path, mutex); @Before public void setUp() { @@ -51,7 +53,7 @@ public class LockTest { expectedMetrics.setCumulativeAcquireCount(1); expectedMetrics.setAcquireFailedCount(1); expectedMetrics.setCumulativeAcquireFailedCount(1); - assertLockMetrics(expectedMetrics); + assertLockMetricsIs(expectedMetrics); List<LockAttempt> slowLockAttempts = LockStats.getGlobal().getLockAttemptSamples(); assertEquals(1, slowLockAttempts.size()); @@ -69,8 +71,15 @@ public class LockTest { assertEquals(0, threadLockStats.getOngoingLockAttempts().size()); } - private void assertLockMetrics(LockMetrics expected) { - LockMetrics actual = LockStats.getGlobal().getLockMetricsByPath().get(lockPath); + private void assertLock2MetricsIs(LockMetrics expected) { + assertLockMetrics(expected, LockStats.getGlobal().getLockMetricsByPath().get(lock2Path)); + } + + private void assertLockMetricsIs(LockMetrics expected) { + assertLockMetrics(expected, LockStats.getGlobal().getLockMetricsByPath().get(lockPath)); + } + + private void assertLockMetrics(LockMetrics expected, LockMetrics actual) { assertNotNull(actual); assertEquals(expected.getCumulativeAcquireCount(), actual.getCumulativeAcquireCount()); @@ -79,6 +88,7 @@ public class LockTest { assertEquals(expected.getCumulativeAcquireSucceededCount(), actual.getCumulativeAcquireSucceededCount()); assertEquals(expected.getCumulativeReleaseCount(), actual.getCumulativeReleaseCount()); assertEquals(expected.getCumulativeReleaseFailedCount(), actual.getCumulativeReleaseFailedCount()); + assertEquals(expected.getCumulativeReentryCount(), actual.getCumulativeReentryCount()); assertEquals(expected.getAndResetAcquireCount(), actual.getAndResetAcquireCount()); assertEquals(expected.getAndResetAcquireFailedCount(), actual.getAndResetAcquireFailedCount()); @@ -86,6 +96,7 @@ public class LockTest { assertEquals(expected.getAndResetAcquireSucceededCount(), actual.getAndResetAcquireSucceededCount()); assertEquals(expected.getAndResetReleaseCount(), actual.getAndResetReleaseCount()); assertEquals(expected.getAndResetReleaseFailedCount(), actual.getAndResetReleaseFailedCount()); + assertEquals(expected.getAndResetReentryCount(), actual.getAndResetReentryCount()); } @Test @@ -104,7 +115,7 @@ public class LockTest { expectedMetrics.setCumulativeAcquireCount(1); expectedMetrics.setAcquireTimedOutCount(1); expectedMetrics.setCumulativeAcquireTimedOutCount(1); - assertLockMetrics(expectedMetrics); + assertLockMetricsIs(expectedMetrics); } @Test @@ -112,36 +123,53 @@ public class LockTest { when(mutex.acquire(anyLong(), any())).thenReturn(true); lock.acquire(acquireTimeout); + assertLockMetricsIs(new LockMetrics() + .setAcquireCount(1) + .setCumulativeAcquireCount(1) + .setAcquireSucceededCount(1) + .setCumulativeAcquireSucceededCount(1)); + + // reenter lock + { + // NB: non-cumulative counters are reset on fetch + lock.acquire(acquireTimeout); + assertLockMetricsIs(new LockMetrics() + .setReentryCount(1) + .setCumulativeAcquireCount(1) + .setCumulativeAcquireSucceededCount(1) + .setCumulativeReentryCount(1)); + + lock.close(); + assertLockMetricsIs(new LockMetrics() + .setCumulativeAcquireCount(1) + .setCumulativeAcquireSucceededCount(1) + .setCumulativeReentryCount(1)); + } - var expectedMetrics = new LockMetrics(); - expectedMetrics.setAcquireCount(1); - expectedMetrics.setCumulativeAcquireCount(1); - expectedMetrics.setAcquireSucceededCount(1); - expectedMetrics.setCumulativeAcquireSucceededCount(1); - assertLockMetrics(expectedMetrics); - - // reenter - // NB: non-cumulative counters are reset on fetch - lock.acquire(acquireTimeout); - expectedMetrics.setAcquireCount(1); // reset to 0 above, + 1 - expectedMetrics.setCumulativeAcquireCount(2); - expectedMetrics.setAcquireSucceededCount(1); // reset to 0 above, +1 - expectedMetrics.setCumulativeAcquireSucceededCount(2); - assertLockMetrics(expectedMetrics); - - // inner-most closes - lock.close(); - expectedMetrics.setAcquireCount(0); // reset to 0 above - expectedMetrics.setAcquireSucceededCount(0); // reset to 0 above - expectedMetrics.setReleaseCount(1); - expectedMetrics.setCumulativeReleaseCount(1); - assertLockMetrics(expectedMetrics); + // nested lock2 + { + lock2.acquire(acquireTimeout); + assertLock2MetricsIs(new LockMetrics() + .setAcquireCount(1) + .setCumulativeAcquireCount(1) + .setAcquireSucceededCount(1) + .setCumulativeAcquireSucceededCount(1)); + + lock2.close(); + assertLock2MetricsIs(new LockMetrics() + .setReleaseCount(1) + .setCumulativeAcquireCount(1) + .setCumulativeAcquireSucceededCount(1) + .setCumulativeReleaseCount(1)); + } - // outer-most closes lock.close(); - expectedMetrics.setReleaseCount(1); // reset to 0 above, +1 - expectedMetrics.setCumulativeReleaseCount(2); - assertLockMetrics(expectedMetrics); + assertLockMetricsIs(new LockMetrics() + .setReleaseCount(1) + .setCumulativeAcquireCount(1) + .setCumulativeAcquireSucceededCount(1) + .setCumulativeReentryCount(1) + .setCumulativeReleaseCount(1)); } @Test |