diff options
75 files changed, 207 insertions, 449 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 7c805417739..f2d63f3bba4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -102,21 +102,6 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP } /** - * Returns the Service with the given id, or null if no such - * configId exists or if it belongs to a non-Service ConfigProducer. - * - * @param configId The configId, e.g. "search.0/tld.0" - * @return Service with the given configId - */ - public Service getService(String configId) { - ConfigProducer cp = getConfigProducer(configId); - if (cp == null || !(cp instanceof Service)) { - return null; - } - return (Service) cp; - } - - /** * Adds the descendant (at any depth level), so it can be looked up * on configId in the Map. * diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java index 2685570b444..051591baa75 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java @@ -51,7 +51,7 @@ public abstract class ConfigModel { * * @param configModelRepo The ConfigModelRepo of the system model */ - public void prepare(ConfigModelRepo configModelRepo, DeployState deployState) { return; } + public void prepare(ConfigModelRepo configModelRepo, DeployState deployState) { } /** * <p>Returns whether this model must be maintained in memory for serving config requests. diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java index 45f64182b2a..8c72b5c0237 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java @@ -88,10 +88,9 @@ public abstract class ConfigModelBuilder<MODEL extends ConfigModel> extends Abst @Override public boolean equals(Object other) { - if (!(other instanceof ConfigModelBuilder)) { + if (!(other instanceof ConfigModelBuilder<?> otherBuilder)) { return false; } - ConfigModelBuilder<?> otherBuilder = (ConfigModelBuilder<?>) other; List<ConfigModelId> thisIds = this.handlesElements(); List<ConfigModelId> otherIds = otherBuilder.handlesElements(); if (thisIds.size() != otherIds.size()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java index cd283866550..e1970b001e1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java @@ -42,8 +42,7 @@ public class ConfigModelId implements Comparable<ConfigModelId> { @Override public boolean equals(Object object) { - if (!(object instanceof ConfigModelId)) return false; - ConfigModelId other = (ConfigModelId)object; + if (!(object instanceof ConfigModelId other)) return false; return this.name.equals(other.name) && this.version.equals(other.version); } diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java index cd21fccd855..547e81354eb 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.producer; -import com.yahoo.api.annotations.Beta; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; @@ -16,11 +15,8 @@ import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.Monitoring; - import java.io.Serializable; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java index 012bffaf7f6..30f9cd202ff 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java @@ -3,19 +3,14 @@ package com.yahoo.config.model.producer; import com.yahoo.api.annotations.Beta; import com.yahoo.config.model.ApplicationConfigProducerRoot; -import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.utils.FreezableMap; -import java.io.PrintStream; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Superclass for all producers with children. diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 9ee279c68d3..b5999c15fad 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -215,13 +215,6 @@ public class MockApplicationPackage implements ApplicationPackage { return new MockApplicationPackage.Builder().withHosts(emptyHosts).withServices(emptyServices).build(); } - public static ApplicationPackage fromSearchDefinitionDirectory(String dir) { - return new MockApplicationPackage.Builder() - .withEmptyHosts() - .withEmptyServices() - .withSchemaDir(dir).build(); - } - // TODO: It might work to just merge this and the above public static ApplicationPackage fromSearchDefinitionAndRootDirectory(String dir) { return new MockApplicationPackage.Builder() diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 7b2aaa32136..365434b9de5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -7,7 +7,6 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.HostSystem; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java b/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java index 840d781ac9c..234aecc6228 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java @@ -45,8 +45,7 @@ public class ModelBuilderAddingAccessControlFilter } private static void addFilterToContainerCluster(ContainerModel containerModel) { - if (!(containerModel.getCluster() instanceof ApplicationContainerCluster)) return; - ApplicationContainerCluster cluster = (ApplicationContainerCluster) containerModel.getCluster(); + if (!(containerModel.getCluster() instanceof ApplicationContainerCluster cluster)) return; Http http = cluster.getHttp(); if (http.getAccessControl().isPresent()) { Chain<Filter> chain = http.getFilterChains() diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java index fd98d21dcd5..7aca60bb930 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -21,7 +21,6 @@ import java.util.List; * xml string and returns a config producer that can be use to test getConfig. * * @author Ulf Lilleengen - * @since 5.1.20 */ @Beta public class TestDriver { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java index c1fd8e4646d..f243c4635c7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java @@ -13,7 +13,6 @@ import java.util.List; * Test utility class that provides many methods for inspecting the state of a completely built model * * @author Ulf Lilleengen - * @since 5.1 */ @Beta public class TestRoot { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java index c05d7bf4942..24c418a9d2f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java @@ -4,8 +4,6 @@ package com.yahoo.config.model.test; import com.yahoo.collections.CollectionUtil; import com.yahoo.config.model.builder.xml.XmlHelper; import org.w3c.dom.Element; -import org.xml.sax.InputSource; - import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; @@ -35,7 +33,4 @@ public class TestUtil { return String.join("\n", lines); } - private static InputSource inputSource(String str) { - return new InputSource(new StringReader(str)); - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index 125966ae91d..2f704db1862 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -22,11 +22,7 @@ public class HostPorts { public final static int BASE_PORT = 19100; final static int MAX_PORTS = 799; - private DeployLogger deployLogger = new DeployLogger() { - public void log(Level level, String message) { - System.err.println("deploy log["+level+"]: "+message); - } - }; + private DeployLogger deployLogger = (level, message) -> System.err.println("deploy log["+level+"]: "+message); private final Map<Integer, NetworkPortRequestor> portDB = new LinkedHashMap<>(); @@ -98,7 +94,7 @@ public class HostPorts { /** Allocate a specific port number for a service */ public int requireNetworkPort(int port, NetworkPortRequestor service, String suffix) { - reservePort(service, port, suffix); + reservePort(service, port); String servType = service.getServiceType(); String configId = service.getConfigId(); portFinder.use(new NetworkPorts.Allocation(port, servType, configId, suffix)); @@ -119,18 +115,13 @@ public class HostPorts { return requireNetworkPort(port, service, suffix); } - /** Convenience method to allocate a preferred or required port number for a service */ - public int wantNetworkPort(int port, NetworkPortRequestor service, String suffix, boolean forceRequired) { - return forceRequired ? requireNetworkPort(port, service, suffix) : wantNetworkPort(port, service, suffix); - } - /** Allocate a dynamic port number for a service */ public int allocateNetworkPort(NetworkPortRequestor service, String suffix) { String servType = service.getServiceType(); String configId = service.getConfigId(); int fallback = nextAvailableNetworkPort(); int port = portFinder.findPort(new NetworkPorts.Allocation(fallback, servType, configId, suffix), hostname); - reservePort(service, port, suffix); + reservePort(service, port); portFinder.use(new NetworkPorts.Allocation(port, servType, configId, suffix)); return port; } @@ -161,7 +152,7 @@ public class HostPorts { * @param service the service that wishes to reserve the port. * @param port the port to be reserved. */ - void reservePort(NetworkPortRequestor service, int port, String suffix) { + void reservePort(NetworkPortRequestor service, int port) { if (portDB.containsKey(port)) { portAlreadyReserved(service, port); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 4e40bd768bf..dd1579eae17 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -24,7 +24,6 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.provision.AllocatedHosts; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 95ec1431cdd..d6f0df8e051 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -337,6 +337,8 @@ public class VespaMetricSet { private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java index b9ed8a3c97c..36ebbe41637 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -38,7 +38,7 @@ public class UserConfigBuilder { ConfigDefinitionKey key = DomConfigPayloadBuilder.parseConfigName(element); Optional<ConfigDefinition> def = configDefinitionStore.getConfigDefinition(key); - if ( ! def.isPresent()) { // TODO: Fail instead of warn + if (def.isEmpty()) { // TODO: Fail instead of warn logger.logApplicationPackage(Level.WARNING, "Unable to find config definition '" + key.asFileName() + "'. Please ensure that the name is spelled correctly, and that the def file is included in a bundle."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java index 3ad81ef9f57..c6844619457 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder; -import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.model.ApplicationConfigProducerRoot; /** * Base class for classes capable of building vespa model. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java index 5d17619b526..70bb80ec314 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.Tuple2; -import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 3cf8ec7375f..ed53a1d2267 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -38,7 +38,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilderB VIP_HANDLER_BINDING); private final ApplicationContainerCluster cluster; - private OptionalInt portBindingOverride; + private final OptionalInt portBindingOverride; public DomHandlerBuilder(ApplicationContainerCluster cluster) { this(cluster, OptionalInt.empty()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java index 92b24f3f7ac..14a5b13d8d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java @@ -2,18 +2,20 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.application.Xml; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; -import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.routing.ApplicationSpec; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.text.XML; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.vespa.model.routing.Routing; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; - -import java.util.Arrays; import java.util.List; /** @@ -29,7 +31,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { @Override public List<ConfigModelId> handlesElements() { - return Arrays.asList(ConfigModelId.fromName("routing")); + return List.of(ConfigModelId.fromName("routing")); } // Overrides ConfigModelBuilder. @@ -71,7 +73,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @param element The element to base the route config on. */ private static void addRoutingTable(RoutingSpec routing, Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); RoutingTableSpec table = new RoutingTableSpec(element.getAttribute("protocol"), verify); NodeList children = element.getChildNodes(); @@ -94,7 +96,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @return The corresponding route spec. */ private static RouteSpec createRouteSpec(Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); RouteSpec route = new RouteSpec(element.getAttribute("name"), verify); String hops = element.getAttribute("hops"); int from = 0; @@ -123,9 +125,9 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @return The corresponding hop spec. */ private static HopSpec createHopSpec(Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); HopSpec hop = new HopSpec(element.getAttribute("name"), element.getAttribute("selector"), verify); - if (Boolean.valueOf(element.getAttribute("ignore-result"))) { + if (Boolean.parseBoolean(element.getAttribute("ignore-result"))) { hop.setIgnoreResult(true); } NodeList children = element.getElementsByTagName("recipient"); @@ -135,4 +137,8 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { } return hop; } + + private static boolean shouldVerify(Element element) { + return !element.hasAttribute("verify") || Boolean.parseBoolean(element.getAttribute("verify")); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java index 5bb6d8cf6bf..f65b7a62f05 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java @@ -20,8 +20,9 @@ import java.util.regex.Pattern; * Default is seconds. */ public class Duration { - private static Pattern pattern = Pattern.compile("([0-9\\.]+)\\s*([a-z]+)?"); - private static Map<String, Integer> unitMultiplier = new HashMap<>(); + + private static final Pattern pattern = Pattern.compile("([0-9\\.]+)\\s*([a-z]+)?"); + private static final Map<String, Integer> unitMultiplier = new HashMap<>(); static { unitMultiplier.put("s", 1000); unitMultiplier.put("d", 1000 * 3600 * 24); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java index a138ef71b2f..4a3cae37570 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java @@ -30,7 +30,7 @@ public class HostPortsTest { void next_available_baseport_is_BASE_PORT_plus_one_when_one_port_has_been_reserved() { HostPorts host = new HostPorts("myhostname"); MockRoot root = new MockRoot(); - host.reservePort(new TestService(root, 1), HostPorts.BASE_PORT, "foo"); + host.reservePort(new TestService(root, 1), HostPorts.BASE_PORT); assertThat(host.nextAvailableBaseport(1), is(HostPorts.BASE_PORT + 1)); } @@ -40,12 +40,12 @@ public class HostPortsTest { MockRoot root = new MockRoot(); for (int p = HostPorts.BASE_PORT; p < HostPorts.BASE_PORT + HostPorts.MAX_PORTS; p += 2) { - host.reservePort(new TestService(root, 1), p, "foo"); + host.reservePort(new TestService(root, 1), p); } assertThat(host.nextAvailableBaseport(2), is(0)); try { - host.reservePort(new TestService(root, 2), HostPorts.BASE_PORT, "bar"); + host.reservePort(new TestService(root, 2), HostPorts.BASE_PORT); } catch (RuntimeException e) { assertThat(e.getMessage(), containsString("Too many ports are reserved")); } diff --git a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java b/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java index b5f1406ca4c..ed38a4b2ba3 100644 --- a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java @@ -7,6 +7,8 @@ import java.util.List; */ public enum SearchNodeMetrics implements VespaMetrics { + CONTENT_PROTON_CONFIG_GENERATION("content.proton.config.generation", Unit.VERSION, "The oldest config generation used by this search node"), + CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL("content.proton.documentdb.documents.total", Unit.DOCUMENT, "The total number of documents in this documents db (ready + not-ready)"), CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY("content.proton.documentdb.documents.ready", Unit.DOCUMENT, "The number of ready documents in this document db"), CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE("content.proton.documentdb.documents.active", Unit.DOCUMENT, "The number of active / searchable documents in this document db"), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java index 54fda58d19c..c4194315922 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java @@ -39,7 +39,7 @@ public class AthenzClientFactoryMock extends AbstractComponent implements Athenz @Override public ZtsClient createZtsClient() { - return new ZtsClientMock(athenz); + return new ZtsClientMock(athenz, createZmsClient()); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java index d3e74965c4b..3ca0fdd0f23 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java @@ -5,10 +5,12 @@ import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.Identity; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; import com.yahoo.vespa.athenz.client.zts.ZtsClient; @@ -17,6 +19,7 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -27,9 +30,14 @@ public class ZtsClientMock implements ZtsClient { private static final Logger log = Logger.getLogger(ZtsClientMock.class.getName()); private final AthenzDbMock athenz; + private final Optional<ZmsClient> zmsClient; public ZtsClientMock(AthenzDbMock athenz) { + this(athenz, null); + } + public ZtsClientMock(AthenzDbMock athenz, ZmsClient zmsClient) { this.athenz = athenz; + this.zmsClient = Optional.ofNullable(zmsClient); } @Override @@ -98,6 +106,12 @@ public class ZtsClientMock implements ZtsClient { } @Override + public boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity) { + return zmsClient.orElseThrow(UnsupportedOperationException::new) + .hasAccess(resource, action, identity); + } + + @Override public void close() { } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 6a493f3f5ed..65320a25984 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -309,7 +309,7 @@ public class AthenzFacade implements AccessControl { } private boolean lookupAccess(AccessTuple t) { - boolean result = zmsClient.hasAccess(AthenzResourceName.fromString(t.resource), t.action, t.identity); + boolean result = ztsClient.hasAccess(AthenzResourceName.fromString(t.resource), t.action, t.identity); log("getAccess(action=%s, resource=%s, principal=%s) = %b", t.action, t.resource, t.identity, result); return result; } diff --git a/jrt/src/com/yahoo/jrt/MandatoryMethods.java b/jrt/src/com/yahoo/jrt/MandatoryMethods.java index b1355c0fb1e..a73e2bfc6dd 100644 --- a/jrt/src/com/yahoo/jrt/MandatoryMethods.java +++ b/jrt/src/com/yahoo/jrt/MandatoryMethods.java @@ -23,6 +23,7 @@ class MandatoryMethods { parent.addMethod(m); //--------------------------------------------------------------------- m = new Method("frt.rpc.getMethodList", "", "SSS", this::getMethodList); + m.requireCapabilities(CapabilitySet.none()); m.methodDesc("Obtain a list of all available methods"); m.returnDesc(0, "names", "Method names"); m.returnDesc(1, "params", "Method parameter types"); @@ -30,6 +31,7 @@ class MandatoryMethods { parent.addMethod(m); //--------------------------------------------------------------------- m = new Method("frt.rpc.getMethodInfo", "s", "sssSSSS", this::getMethodInfo); + m.requireCapabilities(CapabilitySet.none()); m.methodDesc("Obtain detailed information about a single method"); m.paramDesc (0, "methodName", "The method we want information about"); m.returnDesc(0, "desc", "Description of what the method does"); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index b4fa7d8f887..6afc2039c38 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -29,6 +29,7 @@ import com.yahoo.messagebus.network.NetworkOwner; import com.yahoo.messagebus.routing.Hop; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.security.tls.CapabilitySet; import java.io.PrintWriter; import java.io.StringWriter; @@ -100,6 +101,7 @@ public class RPCNetwork implements Network, MethodHandler { servicePool = new RPCServicePool(this, 4096); Method method = new Method("mbus.getVersion", "", "s", this); + method.requireCapabilities(CapabilitySet.none()); method.methodDesc("Retrieves the message bus version."); method.returnDesc(0, "version", "The message bus version."); orb.addMethod(method); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index ab1c2e70735..ba51bed4407 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -97,7 +97,7 @@ public class MetricsManager { return metricsPackets; } - private MetricsPacket.Builder [] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime, ConsumerId consumerId) { + private MetricsPacket.Builder[] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime, ConsumerId consumerId) { List<MetricsPacket.Builder> builders = getMetricsAsBuilders(services, startTime, consumerId); return builders.toArray(new MetricsPacket.Builder[0]); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java index 821636336a8..21c7c78a224 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java @@ -25,14 +25,10 @@ public class Node { } public Node(String role, String hostname, int port, String path) { - Objects.requireNonNull(role, "Null role is not allowed"); - Objects.requireNonNull(hostname, "Null hostname is not allowed"); - Objects.requireNonNull(path, "Null path is not allowed"); - - this.role = role; - this.hostname = hostname; + this.role = Objects.requireNonNull(role, "Null role is not allowed"); + this.hostname = Objects.requireNonNull(hostname, "Null hostname is not allowed"); this.port = port; - this.path = path; + this.path = Objects.requireNonNull(path, "Null path is not allowed"); metricsUriBase = "http://" + hostname + ":" + port + path; } @@ -55,10 +51,10 @@ public class Node { public int hashCode() { return Objects.hash(role, hostname, port, path); } + @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(role).append(":").append(metricsUriBase); - return sb.toString(); + return role + ":" + metricsUriBase; } + } diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 604e9c46b80..98d03514de0 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -152,11 +152,10 @@ namespace { std::pair<std::string, std::string> getMatchedMetrics(const vespalib::string& config) { - FastOS_ThreadPool pool; TestMetricSet mySet; MetricManager mm; mm.registerMetric(mm.getMetricLock(), mySet.set); - mm.init(ConfigUri(config), pool); + mm.init(ConfigUri(config)); MetricNameVisitor visitor; /** Take a copy to verify clone works. @@ -475,7 +474,6 @@ std::string dumpAllSnapshots(const MetricManager& mm, TEST_F(MetricManagerTest, test_snapshots) { - FastOS_ThreadPool pool; FakeTimer* timer = new FakeTimer(1000); std::unique_ptr<MetricManager::Timer> timerImpl(timer); TestMetricSet mySet; @@ -491,8 +489,7 @@ TEST_F(MetricManagerTest, test_snapshots) "consumer[0].tags[0] snaptest\n" "consumer[1].name log\n" "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n"), - pool); + "consumer[1].tags[0] snaptest\n")); MetricNameVisitor visitor; { MetricLockGuard lockGuard(mm.getMetricLock()); @@ -575,7 +572,6 @@ TEST_F(MetricManagerTest, test_snapshots) TEST_F(MetricManagerTest, test_xml_output) { - FastOS_ThreadPool pool; FakeTimer* timer = new FakeTimer(1000); std::unique_ptr<MetricManager::Timer> timerImpl(timer); MetricManager mm(std::move(timerImpl)); @@ -593,8 +589,7 @@ TEST_F(MetricManagerTest, test_xml_output) "consumer[0].tags[0] snaptest\n" "consumer[1].name log\n" "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n"), - pool); + "consumer[1].tags[0] snaptest\n")); takeSnapshots(mm, 1000); @@ -653,7 +648,6 @@ TEST_F(MetricManagerTest, test_xml_output) TEST_F(MetricManagerTest, test_json_output) { - FastOS_ThreadPool pool; FakeTimer* timer = new FakeTimer(1000); std::unique_ptr<MetricManager::Timer> timerImpl(timer); MetricManager mm(std::move(timerImpl)); @@ -668,8 +662,7 @@ TEST_F(MetricManagerTest, test_json_output) "consumer[1]\n" "consumer[0].name snapper\n" "consumer[0].tags[1]\n" - "consumer[0].tags[0] snaptest\n"), - pool); + "consumer[0].tags[0] snaptest\n")); takeSnapshots(mm, 1000); @@ -743,14 +736,12 @@ namespace { struct MetricSnapshotTestFixture { MetricManagerTest& test; - FastOS_ThreadPool pool; FakeTimer* timer; MetricManager manager; MetricSet& mset; MetricSnapshotTestFixture(MetricManagerTest& callerTest, MetricSet& metricSet) : test(callerTest), - pool(), timer(new FakeTimer(1000)), manager(std::unique_ptr<MetricManager::Timer>(timer)), mset(metricSet) @@ -765,8 +756,7 @@ struct MetricSnapshotTestFixture "consumer[1]\n" "consumer[0].name snapper\n" "consumer[0].addedmetrics[1]\n" - "consumer[0].addedmetrics[0] *\n"), - pool); + "consumer[0].addedmetrics[0] *\n")); test.takeSnapshots(manager, 1000); } @@ -986,7 +976,6 @@ TEST_F(MetricManagerTest, json_output_can_have_multiple_sets_with_same_name) TEST_F(MetricManagerTest, test_text_output) { - FastOS_ThreadPool pool; FakeTimer* timer = new FakeTimer(1000); std::unique_ptr<MetricManager::Timer> timerImpl(timer); MetricManager mm(std::move(timerImpl)); @@ -1010,8 +999,7 @@ TEST_F(MetricManagerTest, test_text_output) "consumer[0].tags[0] snaptest\n" "consumer[1].name log\n" "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n"), - pool); + "consumer[1].tags[0] snaptest\n")); std::string expected( "snapshot \"Active metrics showing updates since last snapshot\" from 1000 to 0 period 0\n" "temp.val6 average=2 last=2 min=2 max=2 count=1 total=2\n" @@ -1085,7 +1073,6 @@ TEST_F(MetricManagerTest, test_update_hooks) { std::mutex output_mutex; std::ostringstream output; - FastOS_ThreadPool pool; FakeTimer* timer = new FakeTimer(1000); std::unique_ptr<MetricManager::Timer> timerImpl(timer); // Add a metric set just so one exist @@ -1114,8 +1101,7 @@ TEST_F(MetricManagerTest, test_update_hooks) "consumer[0].tags[0] snaptest\n" "consumer[1].name log\n" "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n"), - pool); + "consumer[1].tags[0] snaptest\n")); output << "Init done\n"; MyUpdateHook postInitShort(output, output_mutex, "AIS", *timer); diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp index 22eb3587eff..b4eb4a1353c 100644 --- a/metrics/src/tests/snapshottest.cpp +++ b/metrics/src/tests/snapshottest.cpp @@ -176,7 +176,6 @@ TEST_F(SnapshotTest, test_snapshot_two_days) TestMetricSet set("test"); FakeTimer* timer; - FastOS_ThreadPool threadPool; MetricManager mm( std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer)); { @@ -185,7 +184,7 @@ TEST_F(SnapshotTest, test_snapshot_two_days) } mm.init(config::ConfigUri("raw:consumer[1]\n" "consumer[0].name \"log\""), - threadPool, false); + false); tick(mm, timer->_timeInSecs * 1000); for (uint32_t days=0; days<2; ++days) { diff --git a/metrics/src/tests/stresstest.cpp b/metrics/src/tests/stresstest.cpp index e942d47b9de..afabf91d5c9 100644 --- a/metrics/src/tests/stresstest.cpp +++ b/metrics/src/tests/stresstest.cpp @@ -74,25 +74,29 @@ OuterMetricSet::OuterMetricSet(MetricSet* owner) OuterMetricSet::~OuterMetricSet() = default; -struct Hammer : public document::Runnable { +struct Hammer { using UP = std::unique_ptr<Hammer>; OuterMetricSet& _metrics; - - Hammer(OuterMetricSet& metrics,FastOS_ThreadPool& threadPool) - : _metrics(metrics) + std::atomic<bool> _stop_requested; + std::thread _thread; + + Hammer(OuterMetricSet& metrics) + : _metrics(metrics), + _stop_requested(false), + _thread() { - start(threadPool); + _thread = std::thread([this](){run();}); } ~Hammer() { - stop(); - join(); + _stop_requested = true; + _thread.join(); //std::cerr << "Loadgiver thread joined\n"; } - void run() override { + void run() { uint64_t i = 0; - while (running()) { + while (!_stop_requested.load(std::memory_order_relaxed)) { ++i; setMetrics(i, _metrics._inner1); setMetrics(i + 3, _metrics._inner2); @@ -114,10 +118,9 @@ TEST(StressTest, test_stress) OuterMetricSet metrics; LOG(info, "Starting load givers"); - FastOS_ThreadPool threadPool; std::vector<Hammer::UP> hammers; for (uint32_t i=0; i<10; ++i) { - hammers.push_back(std::make_unique<Hammer>(metrics, threadPool)); + hammers.push_back(std::make_unique<Hammer>(metrics)); } LOG(info, "Waiting to let loadgivers hammer a while"); std::this_thread::sleep_for(5s); diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index ae75968e605..a0e44ddbeac 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -82,7 +82,9 @@ MetricManager::MetricManager(std::unique_ptr<Timer> timer) _snapshotHookLatency("snapshothooklatency", {}, "Time in ms used to update a single snapshot hook", &_metricManagerMetrics), _resetLatency("resetlatency", {}, "Time in ms used to reset all metrics.", &_metricManagerMetrics), _snapshotLatency("snapshotlatency", {}, "Time in ms used to take a snapshot", &_metricManagerMetrics), - _sleepTimes("sleeptime", {}, "Time in ms worker thread is sleeping", &_metricManagerMetrics) + _sleepTimes("sleeptime", {}, "Time in ms worker thread is sleeping", &_metricManagerMetrics), + _stop_requested(false), + _thread() { registerMetric(getMetricLock(), _metricManagerMetrics); } @@ -95,15 +97,14 @@ MetricManager::~MetricManager() void MetricManager::stop() { - if (!running()) { - return; // Let stop() be idempotent. - } - Runnable::stop(); + request_stop(); { MetricLockGuard sync(_waiter); _cond.notify_all(); } - join(); + if (_thread.joinable()) { + _thread.join(); + } } void @@ -161,7 +162,7 @@ MetricManager::isInitialized() const { } void -MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, bool startThread) +MetricManager::init(const config::ConfigUri & uri, bool startThread) { if (isInitialized()) { throw vespalib::IllegalStateException( @@ -175,7 +176,7 @@ MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, bool configure(getMetricLock(), _configHandle->getConfig()); LOG(debug, "Starting worker thread, waiting for first iteration to complete."); if (startThread) { - Runnable::start(pool); + _thread = std::thread([this](){run();}); // Wait for first iteration to have completed, such that it is safe // to access snapshots afterwards. MetricLockGuard sync(_waiter); @@ -763,7 +764,7 @@ MetricManager::run() } // Ensure correct time for first snapshot _snapshots[0]->getSnapshot().setToTime(currentTime); - while (!stopping()) { + while (!stop_requested()) { currentTime = _timer->getTime(); time_t next = tick(sync, currentTime); if (currentTime < next) { diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 5f35c349f7f..b1777a1d228 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -49,7 +49,6 @@ #include "valuemetric.h" #include "updatehook.h" #include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/jsonwriter.h> #include <vespa/metrics/config-metricsmanager.h> #include <vespa/config/subscription/configsubscriber.h> @@ -61,7 +60,7 @@ template class vespalib::hash_set<metrics::Metric::String>; namespace metrics { -class MetricManager : private document::Runnable +class MetricManager { public: @@ -119,10 +118,15 @@ private: LongAverageMetric _resetLatency; LongAverageMetric _snapshotLatency; LongAverageMetric _sleepTimes; + std::atomic<bool> _stop_requested; + std::thread _thread; + void request_stop() { _stop_requested.store(true, std::memory_order_relaxed); } + bool stop_requested() const { return _stop_requested.load(std::memory_order_relaxed); } + public: MetricManager(std::unique_ptr<Timer> timer = std::make_unique<Timer>()); - ~MetricManager() override; + ~MetricManager(); void stop(); @@ -194,7 +198,7 @@ public: * of consumers. readConfig() will start a config subscription. It should * not be called multiple times. */ - void init(const config::ConfigUri & uri, FastOS_ThreadPool&, bool startThread = true); + void init(const config::ConfigUri & uri, bool startThread = true); /** * Visit a given snapshot for a given consumer. (Empty consumer name means @@ -271,7 +275,7 @@ private: friend struct SnapshotTest; void configure(const MetricLockGuard & guard, std::unique_ptr<MetricsmanagerConfig> conf); - void run() override; + void run(); time_t tick(const MetricLockGuard & guard, time_t currentTime); /** * Utility function for updating periodic metrics. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index ea4944c2bd5..ea35f1e85ff 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -57,7 +57,7 @@ public class Cluster { this.suggested = Objects.requireNonNull(suggested); Objects.requireNonNull(target); if (target.resources().isPresent() && ! target.resources().get().isWithin(minResources, maxResources)) - this.target = Autoscaling.empty(); + this.target = target.withResources(Optional.empty()); // Delete illegal target else this.target = target; this.bcpGroupInfo = Objects.requireNonNull(bcpGroupInfo); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index a68a667ffe6..3825309e97b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -57,6 +57,10 @@ public class Autoscaling { return new Autoscaling(status, description, resources, at, peak, ideal, metrics); } + public Autoscaling withResources(Optional<ClusterResources> resources) { + return new Autoscaling(status, description, resources, at, peak, ideal, metrics); + } + /** Converts this autoscaling into an ideal one at the completion of it. */ public Autoscaling asIdeal(Instant at) { return new Autoscaling(Status.ideal, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index a30c9b588c2..264664f91b2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -119,7 +119,7 @@ public class ClusterModel { /** Returns the relative load adjustment that should be made to this cluster given available measurements. */ public Load loadAdjustment() { - if (nodeTimeseries().isEmpty()) return Load.one(); + if (nodeTimeseries().measurementsPerNode() < 0.5) return Load.one(); // Don't change based on very little data Load adjustment = peakLoad().divide(idealLoad()); if (! safeToScaleDown()) adjustment = adjustment.map(v -> v < 1 ? 1 : v); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index b86a24af5c9..6c02f7466eb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -6,10 +6,8 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Duration; import java.util.List; -import java.util.Optional; import java.util.OptionalDouble; import java.util.function.Predicate; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.autoscale.ClusterModel.warmupDuration; @@ -28,10 +26,10 @@ public class ClusterNodesTimeseries { public ClusterNodesTimeseries(Duration period, Cluster cluster, NodeList clusterNodes, MetricsDb db) { this.clusterNodes = clusterNodes; - // See warmupSeconds*4 into the past to see any generation change in it + // See warmupDuration*4 into the past to see any generation change in it. // If none can be detected we assume the node is new/was down. // If either this is the case, or there is a generation change, we ignore - // the first warmupWindow metrics + // the first warmupWindow metrics. var timeseries = db.getNodeTimeseries(period.plus(warmupDuration.multipliedBy(4)), clusterNodes); if (cluster.lastScalingEvent().isPresent()) { long currentGeneration = cluster.lastScalingEvent().get().generation(); @@ -52,42 +50,15 @@ public class ClusterNodesTimeseries { } /** Returns the average number of measurements per node */ - public int measurementsPerNode() { + public double measurementsPerNode() { if (clusterNodes.size() == 0) return 0; int measurementCount = timeseries.stream().mapToInt(m -> m.size()).sum(); - return measurementCount / clusterNodes.size(); + return (double)measurementCount / clusterNodes.size(); } /** Returns the number of nodes measured in this */ public int nodesMeasured() { return timeseries.size(); } - /** Returns the average load after the given instant */ - public Load averageLoad() { - Load total = Load.zero(); - int count = 0; - for (var nodeTimeseries : timeseries) { - for (var snapshot : nodeTimeseries.asList()) { - total = total.add(snapshot.load()); - count++; - } - } - return total.divide(count); - } - - /** Returns average of the latest load reading from each node */ - public Load currentLoad() { - Load total = Load.zero(); - int count = 0; - for (var nodeTimeseries : timeseries) { - Optional<NodeMetricSnapshot> last = nodeTimeseries.last(); - if (last.isEmpty()) continue; - - total = total.add(last.get().load()); - count++; - } - return total.divide(count); - } - /** * Returns the "peak load" in this: Which is for each load dimension, * the average of the highest reading for that dimension on each node. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index dc0327c9537..428ea784115 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -162,7 +162,8 @@ public class MetricsResponse { generation { // application config generation active on the node @Override - public List<String> metricResponseNames() { return List.of("application_generation"); } + public List<String> metricResponseNames() { return List.of("application_generation", + "content.proton.config.generation"); } @Override double computeFinal(ListMap<String, Double> values) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java index c25b0684f5a..17444ef9d2e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Optional; import java.util.OptionalDouble; import java.util.function.Predicate; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.autoscale.ClusterModel.warmupDuration; @@ -98,7 +97,6 @@ public class NodeTimeseries { } private boolean onAtLeastGeneration(long generation, NodeMetricSnapshot snapshot) { - if (snapshot.generation() < 0) return true; // Content nodes do not yet send generation return snapshot.generation() >= generation; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 9134c376f38..5690a685345 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -126,7 +126,7 @@ public class Nodes { illegal("Cannot add " + node + ": Child nodes need to be allocated"); Optional<Node> existing = node(node.hostname()); if (existing.isPresent()) - illegal("Cannot add " + node + ": A node with this name already exists"); + throw new IllegalStateException("Cannot add " + node + ": A node with this name already exists"); } return db.addNodesInState(nodes.asList(), Node.State.reserved, Agent.system); } @@ -152,7 +152,7 @@ public class Nodes { Optional<Node> existing = node(node.hostname()); if (existing.isPresent()) { if (existing.get().state() != Node.State.deprovisioned) - illegal("Cannot add " + node + ": A node with this name already exists"); + throw new IllegalStateException("Cannot add " + node + ": A node with this name already exists"); node = node.with(existing.get().history()); node = node.with(existing.get().reports()); node = node.with(node.status().withFailCount(existing.get().status().failCount())); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 1d7b7ec7454..43676518330 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -112,8 +112,8 @@ public class NodesV2ApiTest { // POST duplicate node tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); + Request.Method.POST), 500, + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); // DELETE a provisioned node assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index 06df12f73b7..2c47b5162ca 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -254,7 +254,7 @@ App::startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport, int spiProton->createNode(); EV_STARTED("servicelayer"); } else { - proton.getMetricManager().init(identityUri, threadPool); + proton.getMetricManager().init(identityUri); } EV_STARTED("proton"); while (!(SIG::INT.check() || SIG::TERM.check() || (spiProton && spiProton->getNode().attemptedStopped()))) { diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 00f62aefc28..76fc875b209 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -19,6 +19,7 @@ #include <vespa/vespalib/util/signalhandler.h> #include <iostream> #include <thread> +#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP("vespa-transactionlog-inspect"); diff --git a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp index c2d09fa341b..9e9d51abd57 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp @@ -29,6 +29,7 @@ ContentProtonMetrics::ProtonExecutorMetrics::~ProtonExecutorMetrics() = default; ContentProtonMetrics::ContentProtonMetrics() : metrics::MetricSet("content.proton", {}, "Search engine metrics", nullptr), + configGeneration("config.generation", {}, "The oldest config generation used by this process", this), transactionLog(this), resourceUsage(this), executor(this), diff --git a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h index 127e32ada07..c82a6804380 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h @@ -41,6 +41,7 @@ struct ContentProtonMetrics : metrics::MetricSet ~SessionCacheMetrics() override; }; + metrics::LongValueMetric configGeneration; TransLogServerMetrics transactionLog; ResourceUsageMetrics resourceUsage; ProtonExecutorMetrics executor; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 54009ef60f4..34961414d57 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -796,6 +796,7 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) { { ContentProtonMetrics &metrics = _metricsEngine->root(); + metrics.configGeneration.set(getConfigGeneration()); auto tls = _tls->getTransLogServer(); if (tls) { metrics.transactionLog.update(tls->getDomainStats()); diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index af214c34be8..a1a42b592b2 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/fnet/transport.h> #include <vespa/fastos/file.h> +#include <vespa/fastos/thread.h> #include <thread> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 98a9568e4e8..c96b0cdcd61 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -99,7 +99,7 @@ TransLogServer::TransLogServer(FNET_Transport & transport, const vespalib::strin _baseDir(baseDir), _domainConfig(cfg), _executor(maxThreads, CpuUsage::wrap(tls_executor, CpuUsage::Category::WRITE)), - _threadPool(std::make_unique<FastOS_ThreadPool>()), + _thread(), _supervisor(std::make_unique<FRT_Supervisor>(&transport)), _domains(), _reqQ(), @@ -143,25 +143,24 @@ TransLogServer::TransLogServer(FNET_Transport & transport, const vespalib::strin } else { throw std::runtime_error(make_string("Failed creating tls base dir %s r(%d), e(%d). Requires manual intervention.", _baseDir.c_str(), retval, errno)); } - start(*_threadPool); + _thread = std::thread([this](){run();}); } TransLogServer::~TransLogServer() { - _closed = true; - stop(); - join(); + request_stop(); + _thread.join(); _executor.sync(); _executor.shutdown(); _executor.sync(); } -bool -TransLogServer::onStop() +void +TransLogServer::request_stop() { + _closed = true; LOG(info, "Stopping TLS"); _reqQ.push(nullptr); - return true; } void diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h index f7ea80c9248..2c5fbf51a08 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h @@ -2,7 +2,6 @@ #pragma once #include "domainconfig.h" -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/document/util/queue.h> #include <vespa/fnet/frt/invokable.h> @@ -18,7 +17,7 @@ namespace search::transactionlog { class TransLogServerExplorer; class Domain; -class TransLogServer : private FRT_Invokable, public document::Runnable, public WriterFactory +class TransLogServer : private FRT_Invokable, public WriterFactory { public: friend class TransLogServerExplorer; @@ -36,8 +35,8 @@ public: TransLogServer & setDomainConfig(const DomainConfig & cfg); private: - bool onStop() override; - void run() override; + void request_stop(); + void run(); void exportRPC(FRT_Supervisor & supervisor); void relayToThreadRPC(FRT_RPCRequest *req); @@ -63,11 +62,13 @@ private: using ReadGuard = std::shared_lock<std::shared_mutex>; using WriteGuard = std::unique_lock<std::shared_mutex>; + bool running() const { return !_closed.load(std::memory_order_relaxed); } + vespalib::string _name; vespalib::string _baseDir; DomainConfig _domainConfig; vespalib::ThreadStackExecutor _executor; - std::unique_ptr<FastOS_ThreadPool> _threadPool; + std::thread _thread; std::unique_ptr<FRT_Supervisor> _supervisor; DomainList _domains; mutable std::shared_mutex _domainMutex;; // Protects _domains diff --git a/searchlib/src/vespa/searchlib/util/runnable.h b/searchlib/src/vespa/searchlib/util/runnable.h index e268b13e09a..4b353209762 100644 --- a/searchlib/src/vespa/searchlib/util/runnable.h +++ b/searchlib/src/vespa/searchlib/util/runnable.h @@ -4,6 +4,7 @@ #include <mutex> #include <condition_variable> +#include <vespa/fastos/thread.h> namespace search { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java b/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java index 8fa077027a9..010b8a5b228 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java @@ -44,7 +44,8 @@ public class CapabilitySet implements ToCapabilitySet { SHARED_CAPABILITIES_APP_NODE); public static final CapabilitySet CONTAINER_NODE = predefined( "vespa.container_node", - Capability.CONTENT__DOCUMENT_API, Capability.CONTENT__SEARCH_API, SHARED_CAPABILITIES_APP_NODE); + Capability.CONTAINER__DOCUMENT_API, Capability.CONTENT__DOCUMENT_API, Capability.CONTENT__SEARCH_API, + SHARED_CAPABILITIES_APP_NODE); public static final CapabilitySet CLUSTER_CONTROLLER_NODE = predefined( "vespa.cluster_controller_node", Capability.CONTENT__CLUSTER_CONTROLLER__INTERNAL_STATE_API, diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index 582e6957c22..3a16ee170fe 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/storage/bucketdb/btree_lockable_map.hpp> #include <vespa/storage/bucketdb/striped_btree_lockable_map.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 97d1c22364f..78fa32e24e5 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -100,7 +100,7 @@ void MetricsTest::SetUp() { _visitorMetrics = std::make_shared<VisitorMetrics>(); _visitorMetrics->initThreads(4); _topSet->registerMetric(*_visitorMetrics); - _metricManager->init(config::ConfigUri(_config->getConfigId()), _node->getThreadPool()); + _metricManager->init(config::ConfigUri(_config->getConfigId())); } void MetricsTest::TearDown() { diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 50ad7b54382..7f3fe06fc29 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -533,17 +533,18 @@ TEST_F(FileStorManagerTest, handler_priority) { ASSERT_EQ(75, filestorHandler.getNextMessage(stripeId).msg->getPriority()); } -class MessagePusherThread : public document::Runnable { +class MessagePusherThread { public: FileStorHandler& _handler; Document::SP _doc; std::atomic<bool> _done; std::atomic<bool> _threadDone; - + std::thread _thread; + MessagePusherThread(FileStorHandler& handler, Document::SP doc); - ~MessagePusherThread() override; + ~MessagePusherThread(); - void run() override { + void run() { while (!_done) { document::BucketIdFactory factory; document::BucketId bucket(16, factory.getBucketId(_doc->getId()).getRawId()); @@ -558,11 +559,16 @@ public: }; MessagePusherThread::MessagePusherThread(FileStorHandler& handler, Document::SP doc) - : _handler(handler), _doc(std::move(doc)), _done(false), _threadDone(false) -{} -MessagePusherThread::~MessagePusherThread() = default; + : _handler(handler), _doc(std::move(doc)), _done(false), _threadDone(false), _thread() +{ + _thread = std::thread([this](){run();}); +} +MessagePusherThread::~MessagePusherThread() +{ + _thread.join(); +} -class MessageFetchingThread : public document::Runnable { +class MessageFetchingThread { public: const uint32_t _threadId; FileStorHandler& _handler; @@ -571,13 +577,17 @@ public: std::atomic<bool> _done; std::atomic<bool> _failed; std::atomic<bool> _threadDone; - + std::thread _thread; + explicit MessageFetchingThread(FileStorHandler& handler) : _threadId(0), _handler(handler), _config(0), _fetchedCount(0), _done(false), - _failed(false), _threadDone(false) - {} - - void run() override { + _failed(false), _threadDone(false), _thread() + { + _thread = std::thread([this](){run();}); + } + ~MessageFetchingThread(); + + void run() { while (!_done) { FileStorHandler::LockedMessage msg = _handler.getNextMessage(_threadId); if (msg.msg.get()) { @@ -596,6 +606,10 @@ public: _threadDone = true; }; }; +MessageFetchingThread::~MessageFetchingThread() +{ + _thread.join(); +} TEST_F(FileStorManagerTest, handler_paused_multi_thread) { FileStorHandlerComponents c(*this); @@ -606,12 +620,8 @@ TEST_F(FileStorManagerTest, handler_paused_multi_thread) { Document::SP doc(createDocument(content, "id:footype:testdoctype1:n=1234:bar").release()); - FastOS_ThreadPool pool; MessagePusherThread pushthread(filestorHandler, doc); - pushthread.start(pool); - MessageFetchingThread thread(filestorHandler); - thread.start(pool); for (uint32_t i = 0; i < 50; ++i) { std::this_thread::sleep_for(2ms); diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index b7903de0fe2..1fb5a9730c4 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -97,7 +97,7 @@ void StateReporterTest::SetUp() { _filestorMetrics->initDiskMetrics(1, 1); _topSet->registerMetric(*_filestorMetrics); - _metricManager->init(config::ConfigUri(_config->getConfigId()), _node->getThreadPool()); + _metricManager->init(config::ConfigUri(_config->getConfigId())); } void StateReporterTest::TearDown() { diff --git a/storage/src/vespa/storage/common/storagelinkqueued.h b/storage/src/vespa/storage/common/storagelinkqueued.h index 74434c0116b..17a344a368a 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.h +++ b/storage/src/vespa/storage/common/storagelinkqueued.h @@ -16,7 +16,6 @@ #include "storagelink.h" #include <vespa/storageframework/generic/thread/runnable.h> -#include <vespa/vespalib/util/document_runnable.h> #include <deque> #include <limits> #include <mutex> diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 08a48cc2d8a..99f61c62cd1 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -10,7 +10,6 @@ #include "filestorhandler.h" #include "service_layer_host_info_reporter.h" -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/persistence/spi/bucketexecutor.h> diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 593640e7d03..156ec8bc031 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -23,7 +23,6 @@ #include <vespa/messagebus/imessagehandler.h> #include <vespa/messagebus/ireplyhandler.h> #include <vespa/config/helper/ifetchercallback.h> -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/config/subscription/configuri.h> #include <vespa/config-bucketspaces.h> #include <map> diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index 0adfb209e66..dddcd42aad7 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -15,7 +15,6 @@ #include <vespa/storageframework/generic/thread/runnable.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/document/bucket/bucket.h> -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/metrics/metricset.h> #include <vespa/metrics/summetric.h> #include <vespa/metrics/countmetric.h> diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 2836ab80acf..a09abb25f7a 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -201,7 +201,7 @@ StorageNode::initialize() // have been created, such that we don't need to pay the extra cost of // reinitializing metric manager often. if ( ! _context.getComponentRegister().getMetricManager().isInitialized() ) { - _context.getComponentRegister().getMetricManager().init(_configUri, _context.getThreadPool()); + _context.getComponentRegister().getMetricManager().init(_configUri); } if (_chain) { diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.h b/storage/src/vespa/storage/storageserver/storagenodecontext.h index f07bdd37cd4..52709fb1d9b 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.h +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.h @@ -28,12 +28,6 @@ struct StorageNodeContext { */ ComponentRegister& getComponentRegister() { return *_componentRegister; } - /** - * There currently exist threads that doesn't use the component model. - * Let the backend threadpool be accessible for now. - */ - FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } - ~StorageNodeContext(); protected: diff --git a/storage/src/vespa/storage/visiting/visitormanager.h b/storage/src/vespa/storage/visiting/visitormanager.h index 0c5ec08fb4c..02bb37db59f 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.h +++ b/storage/src/vespa/storage/visiting/visitormanager.h @@ -30,7 +30,6 @@ #include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/config/helper/ifetchercallback.h> -#include <vespa/vespalib/util/document_runnable.h> namespace config { class ConfigUri; diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h index 729b675df3a..f6204fed438 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.h +++ b/storage/src/vespa/storage/visiting/visitorthread.h @@ -22,7 +22,6 @@ #include <vespa/storageframework/generic/thread/runnable.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <vespa/metrics/metrictimer.h> -#include <vespa/vespalib/util/document_runnable.h> #include <atomic> #include <deque> diff --git a/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h b/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h index d228dace1ed..1aede4d12e8 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h +++ b/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h @@ -31,7 +31,6 @@ public: virtual ComponentRegisterImpl& getComponentRegister() { return *_compReg; } FakeClock& getClock() { return _clock; } ThreadPoolImpl& getThreadPoolImpl() { return _threadPool; } - FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } }; } diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp index 925c9cda248..314434a4c1a 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp @@ -28,11 +28,11 @@ ThreadImpl::ThreadImpl(ThreadPoolImpl& pool, _tickDataPtr(0), _interrupted(false), _joined(false), - _thread(*this), + _thread(), _cpu_category(cpu_category) { _tickData[load_relaxed(_tickDataPtr)]._lastTick = pool.getClock().getMonotonicTime(); - _thread.start(_pool.getThreadPool()); + _thread = std::thread([this](){run();}); } ThreadImpl::~ThreadImpl() @@ -70,19 +70,21 @@ void ThreadImpl::interrupt() { _interrupted.store(true, std::memory_order_relaxed); - _thread.stop(); } void ThreadImpl::join() { - _thread.join(); + if (_thread.joinable()) { + _thread.join(); + } } vespalib::string ThreadImpl::get_live_thread_stack_trace() const { - return vespalib::SignalHandler::get_cross_thread_stack_trace(_thread.native_thread_id()); + auto native_handle = const_cast<std::thread&>(_thread).native_handle(); + return vespalib::SignalHandler::get_cross_thread_stack_trace(native_handle); } void diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h index d95ba2a37ef..68ed63ea17c 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h @@ -4,7 +4,6 @@ #include <vespa/storageframework/generic/thread/thread.h> #include <vespa/vespalib/util/cpu_usage.h> -#include <vespa/vespalib/util/document_runnable.h> #include <array> #include <atomic> #include <optional> @@ -15,12 +14,6 @@ struct ThreadPoolImpl; class ThreadImpl final : public Thread { - struct BackendThread : public document::Runnable { - ThreadImpl& _impl; - explicit BackendThread(ThreadImpl& impl) : _impl(impl) {} - void run() override { _impl.run(); } - }; - /** * Internal data race free implementation of tick data that maps to and * from ThreadTickData. We hide the atomicity of this since atomic vars @@ -52,7 +45,7 @@ class ThreadImpl final : public Thread std::atomic<uint32_t> _tickDataPtr; std::atomic<bool> _interrupted; bool _joined; - BackendThread _thread; + std::thread _thread; std::optional<vespalib::CpuUsage::Category> _cpu_category; void run(); diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp index 95959d06b54..068de8f5880 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp @@ -15,8 +15,7 @@ using vespalib::IllegalStateException; namespace storage::framework::defaultimplementation { ThreadPoolImpl::ThreadPoolImpl(Clock& clock) - : _backendThreadPool(std::make_unique<FastOS_ThreadPool>()), - _clock(clock), + : _clock(clock), _stopping(false) { } @@ -44,7 +43,6 @@ ThreadPoolImpl::~ThreadPoolImpl() } std::this_thread::sleep_for(10ms); } - _backendThreadPool->Close(); } Thread::UP diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h index b788a3eed78..07b2dd78ed9 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h @@ -15,7 +15,6 @@ class ThreadImpl; struct ThreadPoolImpl final : public ThreadPool { - std::unique_ptr<FastOS_ThreadPool> _backendThreadPool; std::vector<ThreadImpl*> _threads; mutable std::mutex _threadVectorLock; Clock & _clock; @@ -30,7 +29,6 @@ public: std::optional<vespalib::CpuUsage::Category> cpu_category) override; void visitThreads(ThreadVisitor&) const override; void unregisterThread(ThreadImpl&); - FastOS_ThreadPool& getThreadPool() { return *_backendThreadPool; } Clock& getClock() { return _clock; } }; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index cf46cad57b1..21c8f4ddd31 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.api.ZToken; import com.yahoo.vespa.athenz.client.ErrorHandler; import com.yahoo.vespa.athenz.client.common.ClientBase; +import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.AccessTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.AwsTemporaryCredentialsResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; @@ -221,6 +222,19 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { }); } + @Override + public boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity) { + URI uri = ztsUrl.resolve(String.format("access/%s/%s?principal=%s", + action, resource.toResourceNameString(), identity.getFullName())); + HttpUriRequest request = RequestBuilder.get() + .setUri(uri) + .build(); + return execute(request, response -> { + AccessResponseEntity result = readEntity(response, AccessResponseEntity.class); + return result.granted; + }); + } + private InstanceIdentity getInstanceIdentity(HttpResponse response) throws IOException { InstanceIdentityCredentials entity = readEntity(response, InstanceIdentityCredentials.class); return entity.getServiceToken() != null diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index c4be6d8ced7..eade6229123 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -5,6 +5,7 @@ import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; @@ -187,5 +188,16 @@ public interface ZtsClient extends AutoCloseable { */ AwsTemporaryCredentials getAwsTemporaryCredentials(AthenzDomain athenzDomain, AwsRole awsRole, Duration duration, String externalId); + /** + * Check access to resource for a given principal + * + * @param resource The resource to verify access to + * @param action Action to verify + * @param identity Principal that requests access + * @return <code>true</code> if access is allowed, <code>false</code> otherwise + */ + boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity); + void close(); + } diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 73e8b93a2ff..c8536fc68c1 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -24,7 +24,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT cpu_usage.cpp crc.cpp destructor_callbacks.cpp - document_runnable.cpp doom.cpp dual_merge_director.cpp error.cpp diff --git a/vespalib/src/vespa/vespalib/util/document_runnable.cpp b/vespalib/src/vespa/vespalib/util/document_runnable.cpp deleted file mode 100644 index c0af72dbbb1..00000000000 --- a/vespalib/src/vespa/vespalib/util/document_runnable.cpp +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "document_runnable.h" -#include <vespa/vespalib/util/exceptions.h> -#include <cassert> - -namespace document { - -Runnable::Runnable() - : _stateLock(), - _stateCond(), - _state(NOT_RUNNING) -{ -} - -Runnable::~Runnable() { - std::lock_guard monitorGuard(_stateLock); - assert(getState() == NOT_RUNNING); -} - -bool Runnable::start(FastOS_ThreadPool& pool) -{ - std::unique_lock guard(_stateLock); - _stateCond.wait(guard, [&](){ return (getState() != STOPPING);}); - - if (getState() != NOT_RUNNING) return false; - set_state(STARTING); - if (pool.NewThread(this) == nullptr) { - throw vespalib::IllegalStateException("Failed starting a new thread", VESPA_STRLOC); - } - return true; -} - -void Runnable::set_state(State new_state) noexcept -{ - _state.store(new_state, std::memory_order_relaxed); -} - -bool Runnable::stopping() const noexcept -{ - State s(getState()); - return (s == STOPPING) || (s == RUNNING && GetThread()->GetBreakFlag()); -} - -bool Runnable::running() const noexcept -{ - State s(getState()); - // Must check break-flag too, as threadpool will use that to close - // down. - return (s == STARTING || (s == RUNNING && !GetThread()->GetBreakFlag())); -} - -bool Runnable::stop() -{ - std::lock_guard monitor(_stateLock); - if (getState() == STOPPING || getState() == NOT_RUNNING) return false; - GetThread()->SetBreakFlag(); - set_state(STOPPING); - return onStop(); -} - -bool Runnable::onStop() -{ - return true; -} - -bool Runnable::join() const -{ - std::unique_lock guard(_stateLock); - assert ((getState() != STARTING) && (getState() != RUNNING)); - _stateCond.wait(guard, [&](){ return (getState() == NOT_RUNNING);}); - return true; -} - -FastOS_ThreadId Runnable::native_thread_id() const noexcept -{ - return GetThread()->GetThreadId(); -} - -void Runnable::Run(FastOS_ThreadInterface*, void*) -{ - { - std::lock_guard guard(_stateLock); - // Don't set state if its already at stopping. (And let run() be - // called even though about to stop for consistency) - if (getState() == STARTING) { - set_state(RUNNING); - } - } - - // By not catching exceptions, they should abort whole application. - // We should thus not need to have a catch-all to set state to not - // running. - run(); - - { - std::lock_guard guard(_stateLock); - set_state(NOT_RUNNING); - _stateCond.notify_all(); - } -} - -} diff --git a/vespalib/src/vespa/vespalib/util/document_runnable.h b/vespalib/src/vespa/vespalib/util/document_runnable.h deleted file mode 100644 index 89388bac34c..00000000000 --- a/vespalib/src/vespa/vespalib/util/document_runnable.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class document::Runnable - * @ingroup util - * - * @brief Implementation of FastOS_Runnable that implements threadsafe stop. - * - * FastOS_Runnable can easily be used unsafe. If you use the thread pointer for - * anything after your runnable had returned from Run(), it could affect another - * runnable now using that thread. - * - * Using this class should be foolproof to avoid synchronization issues during - * thread starting and stopping :) - * - * @author H�kon Humberset - * @date 2005-09-19 - */ - -#pragma once - -#include <vespa/fastos/thread.h> -#include <atomic> - -namespace document { - -class Runnable : private FastOS_Runnable { -public: - enum State { NOT_RUNNING, STARTING, RUNNING, STOPPING }; - -private: - mutable std::mutex _stateLock; - mutable std::condition_variable _stateCond; - std::atomic<State> _state; - - void Run(FastOS_ThreadInterface*, void*) override; - void set_state(State new_state) noexcept; // _stateLock must be held -public: - /** - * Create a runnable. - * @param pool If set, runnable will be started in constructor. - */ - Runnable(); - ~Runnable() override; - - /** - * Start this runnable. - * @param pool The threadpool from which a thread is acquired. - * @return True if thread was started, false if thread was already running. - */ - bool start(FastOS_ThreadPool& pool); - - /** - * Stop this runnable. - * @return True if thread was stopped, false if thread was not running. - */ - bool stop(); - - /** - * Called in stop(). Implement, to for instance notify any monitors that - * can be waiting. - */ - virtual bool onStop(); - - /** - * Wait for this thread to finish, if it is in the process of stopping. - * @return True if thread finished (or not running), false if thread is - * running normally and no stop is scheduled. - */ - bool join() const; - - /** - * Implement this to make the runnable actually do something. - */ - virtual void run() = 0; - - /** - * Get the current state of this runnable. - * Thread safe (but relaxed) read; may be stale if done outside _stateLock. - */ - [[nodiscard]] State getState() const noexcept { - return _state.load(std::memory_order_relaxed); - } - - /** Check if system is in the process of stopping. */ - [[nodiscard]] bool stopping() const noexcept; - - /** - * Checks if runnable is running or not. (Started is considered running) - */ - [[nodiscard]] bool running() const noexcept; - - FastOS_ThreadId native_thread_id() const noexcept; -}; - -} - |