diff options
47 files changed, 444 insertions, 223 deletions
diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java index 52a37c9489e..9020765f777 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.apps.clustercontroller; import com.google.inject.Inject; @@ -29,11 +29,21 @@ public class ClusterController extends AbstractComponent private final Map<String, FleetController> controllers = new TreeMap<>(); private final Map<String, StatusHandler.ContainerStatusPageServer> status = new TreeMap<>(); + /** + * Dependency injection constructor for controller. {@link ZooKeeperProvider} argument given + * to ensure that zookeeper has started before we start polling it. + */ + @SuppressWarnings("unused") @Inject + public ClusterController(ZooKeeperProvider zooKeeperProvider) { + this(); + } + ClusterController() { metricWrapper = new JDiscMetricWrapper(null); } + public void setOptions(FleetControllerOptions options, Metric metricImpl) throws Exception { metricWrapper.updateMetricImplementation(metricImpl); verifyThatZooKeeperWorks(options); diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/DummyZooKeeperProvider.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/DummyZooKeeperProvider.java new file mode 100644 index 00000000000..f961297643e --- /dev/null +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/DummyZooKeeperProvider.java @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.apps.clustercontroller; + +/** + * A dummy zookeeper provider when we do not run our own zookeeper instance. + * + * @author Ulf Lilleengen + */ +public class DummyZooKeeperProvider implements ZooKeeperProvider { +} diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StandaloneZooKeeperProvider.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StandaloneZooKeeperProvider.java new file mode 100644 index 00000000000..baad012a234 --- /dev/null +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StandaloneZooKeeperProvider.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.apps.clustercontroller; + +import com.yahoo.vespa.zookeeper.VespaZooKeeperServer; + +/** + * ZooKeeper provider that ensures we are running our own instance of zookeeper. + * + * @author Ulf Lilleengen + */ +public class StandaloneZooKeeperProvider implements ZooKeeperProvider { + + public StandaloneZooKeeperProvider(VespaZooKeeperServer server) { + } + +} diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ZooKeeperProvider.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ZooKeeperProvider.java new file mode 100644 index 00000000000..bb18bcc65d6 --- /dev/null +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ZooKeeperProvider.java @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.apps.clustercontroller; + +/** + * Abstraction we can depend on providing us with a zookeeper server being up. + * + * @author Ulf Lilleengen + */ +public interface ZooKeeperProvider { +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index b2aa1e6b704..14fbeb17aaf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -54,11 +54,23 @@ public class ClusterControllerContainer extends Container implements "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler", "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); - addComponent("clustercontroller-zookeeper-server", - runStandaloneZooKeeper - ? "com.yahoo.vespa.zookeeper.VespaZooKeeperServerImpl" - : "com.yahoo.vespa.zookeeper.DummyVespaZooKeeperServer", - ZOOKEEPER_SERVER_BUNDLE); + if (runStandaloneZooKeeper) { + addComponent("clustercontroller-zkrunner", + "com.yahoo.vespa.zookeeper.VespaZooKeeperServerImpl", + ZOOKEEPER_SERVER_BUNDLE); + addComponent("clustercontroller-zkprovider", + "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StandaloneZooKeeperProvider", + CLUSTERCONTROLLER_BUNDLE); + } else { + // TODO bjorncs/jonmv: remove extraneous ZooKeeperProvider layer + addComponent( + "clustercontroller-zkrunner", + "com.yahoo.vespa.zookeeper.DummyVespaZooKeeperServer", + ZOOKEEPER_SERVER_BUNDLE); + addComponent("clustercontroller-zkprovider", + "com.yahoo.vespa.clustercontroller.apps.clustercontroller.DummyZooKeeperProvider", + CLUSTERCONTROLLER_BUNDLE); + } addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, "controller", isHosted)); // TODO: Why are bundles added here instead of in the cluster? @@ -84,7 +96,7 @@ public class ClusterControllerContainer extends Container implements return ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; } - private void addHandler(Handler<?> h, String path) { + private void addHandler(Handler h, String path) { h.addServerBindings(SystemBindingPattern.fromHttpPath(path)); super.addHandler(h); } @@ -104,7 +116,7 @@ public class ClusterControllerContainer extends Container implements } private void addHandler(String id, String className, String path, ComponentSpecification bundle) { - addHandler(new Handler<>(createComponentModel(id, className, bundle)), path); + addHandler(new Handler(createComponentModel(id, className, bundle)), path); } private ReindexingContext reindexingContext() { diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index 65dc800c275..e8ef1765473 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -216,8 +216,7 @@ GetConfig::Main() vespaVersion = VespaVersion::fromString(vespaVersionString); } - int protocolVersion = config::protocol::readProtocolVersion(); - FRTConfigRequestFactory requestFactory(protocolVersion, traceLevel, vespaVersion, config::protocol::readProtocolCompressionType()); + FRTConfigRequestFactory requestFactory(traceLevel, vespaVersion, config::protocol::readProtocolCompressionType()); FRTConnection connection(spec, _server->supervisor(), TimingValues()); ConfigKey key(configId, defName, defNamespace, defMD5, defSchema); ConfigState state(configMD5, generation, false); diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index 0d70605fa62..6cffe079dae 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -218,7 +218,7 @@ namespace { FRTFixture(SourceFixture & f1) : result(2000, 10000), - requestFactory(1, 3, VespaVersion::fromString("1.2.3"), CompressionType::UNCOMPRESSED), + requestFactory(3, VespaVersion::fromString("1.2.3"), CompressionType::UNCOMPRESSED), src(ConnectionFactory::SP(new FactoryMock(&f1.conn)), requestFactory, ConfigAgent::UP(new AgentFixture(&result)), diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp index d32ae411125..fbc13556d14 100644 --- a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp @@ -10,7 +10,7 @@ namespace config { /** * Factory for creating config requests depending on protocol version; */ -FRTConfigRequestFactory::FRTConfigRequestFactory([[maybe_unused]] int protocolVersion, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType) +FRTConfigRequestFactory::FRTConfigRequestFactory(int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType) : _traceLevel(traceLevel), _vespaVersion(vespaVersion), _hostName(vespalib::HostName::get()), @@ -18,8 +18,7 @@ FRTConfigRequestFactory::FRTConfigRequestFactory([[maybe_unused]] int protocolVe { } -FRTConfigRequestFactory::~FRTConfigRequestFactory() { -} +FRTConfigRequestFactory::~FRTConfigRequestFactory() = default; FRTConfigRequest::UP FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.h b/config/src/vespa/config/frt/frtconfigrequestfactory.h index c70b8920fdd..7c37ecd76b2 100644 --- a/config/src/vespa/config/frt/frtconfigrequestfactory.h +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.h @@ -18,7 +18,7 @@ namespace config { class FRTConfigRequestFactory { public: - FRTConfigRequestFactory(int protocolVersion, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType); + FRTConfigRequestFactory(int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType); ~FRTConfigRequestFactory(); FRTConfigRequest::UP createConfigRequest(const ConfigKey & key, Connection * connection, const ConfigState & state, int64_t serverTimeout) const; diff --git a/config/src/vespa/config/frt/frtsourcefactory.cpp b/config/src/vespa/config/frt/frtsourcefactory.cpp index c58571e3ac4..67e973562db 100644 --- a/config/src/vespa/config/frt/frtsourcefactory.cpp +++ b/config/src/vespa/config/frt/frtsourcefactory.cpp @@ -4,9 +4,9 @@ namespace config { -FRTSourceFactory::FRTSourceFactory(ConnectionFactory::UP connectionFactory, const TimingValues & timingValues, int protocolVersion, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType) +FRTSourceFactory::FRTSourceFactory(ConnectionFactory::UP connectionFactory, const TimingValues & timingValues, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType) : _connectionFactory(std::move(connectionFactory)), - _requestFactory(protocolVersion, traceLevel, vespaVersion, compressionType), + _requestFactory(traceLevel, vespaVersion, compressionType), _timingValues(timingValues) { } diff --git a/config/src/vespa/config/frt/frtsourcefactory.h b/config/src/vespa/config/frt/frtsourcefactory.h index 23596c1c6cd..4bbcbcb366d 100644 --- a/config/src/vespa/config/frt/frtsourcefactory.h +++ b/config/src/vespa/config/frt/frtsourcefactory.h @@ -14,7 +14,7 @@ namespace config { class FRTSourceFactory : public SourceFactory { public: - FRTSourceFactory(ConnectionFactory::UP connectionFactory, const TimingValues & timingValues, int protocolVersion, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType); + FRTSourceFactory(ConnectionFactory::UP connectionFactory, const TimingValues & timingValues, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType); /** * Create source handling config described by key. diff --git a/config/src/vespa/config/subscription/sourcespec.cpp b/config/src/vespa/config/subscription/sourcespec.cpp index 326b3191fd0..e355e0c1bd6 100644 --- a/config/src/vespa/config/subscription/sourcespec.cpp +++ b/config/src/vespa/config/subscription/sourcespec.cpp @@ -121,7 +121,7 @@ ServerSpec::createSourceFactory(const TimingValues & timingValues) const { const auto vespaVersion = VespaVersion::getCurrentVersion(); return std::make_unique<FRTSourceFactory>(std::make_unique<FRTConnectionPool>(*this, timingValues), timingValues, - _protocolVersion, _traceLevel, vespaVersion, _compressionType); + _traceLevel, vespaVersion, _compressionType); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 22f92006c6b..824a374642a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -666,7 +666,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse nodes(String tenantName, String applicationName, String instanceName, String environment, String region) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, id); Slime slime = new Slime(); @@ -689,7 +689,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse clusters(String tenantName, String applicationName, String instanceName, String environment, String region) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); Application application = controller.serviceRegistry().configServer().nodeRepository().getApplication(zone, id); Slime slime = new Slime(); @@ -762,7 +762,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, Map<String, String> queryParameters) { ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); DeploymentId deployment = new DeploymentId(application, zone); InputStream logStream = controller.serviceRegistry().configServer().getLogs(deployment, queryParameters); return new HttpResponse(200) { @@ -775,7 +775,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse metrics(String tenantName, String applicationName, String instanceName, String environment, String region) { ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); DeploymentId deployment = new DeploymentId(application, zone); List<ProtonMetrics> protonMetrics = controller.serviceRegistry().configServer().getProtonMetrics(deployment); return buildResponseFromProtonMetrics(protonMetrics); @@ -1090,7 +1090,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .orElseThrow(() -> new NotExistsException(id + " not found")); DeploymentId deploymentId = new DeploymentId(instance.id(), - ZoneId.from(environment, region)); + requireZone(environment, region)); Deployment deployment = instance.deployments().get(deploymentId.zoneId()); if (deployment == null) @@ -1257,7 +1257,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) { Instance instance = controller.applications().requireInstance(ApplicationId.from(tenantName, applicationName, instanceName)); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); Deployment deployment = instance.deployments().get(zone); if (deployment == null) { throw new NotExistsException(instance + " has no deployment in " + zone); @@ -1293,7 +1293,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - ZoneId.from(environment, region)); + requireZone(environment, region)); Slime slime = new Slime(); Cursor array = slime.setObject().setArray("globalrotationoverride"); controller.routing().globalRotationStatus(deploymentId) @@ -1311,7 +1311,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse rotationStatus(String tenantName, String applicationName, String instanceName, String environment, String region, Optional<String> endpointId) { ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); Instance instance = controller.applications().requireInstance(applicationId); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); RotationId rotation = findRotationId(instance, endpointId); Deployment deployment = instance.deployments().get(zone); if (deployment == null) { @@ -1400,7 +1400,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse suspended(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - ZoneId.from(environment, region)); + requireZone(environment, region)); boolean suspended = controller.applications().isSuspended(deploymentId); Slime slime = new Slime(); Cursor response = slime.setObject(); @@ -1410,16 +1410,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); - ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), + ZoneId zone = requireZone(environment, region); + ServiceApiResponse response = new ServiceApiResponse(zone, new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - controller.zoneRegistry().getConfigServerApiUris(ZoneId.from(environment, region)), + controller.zoneRegistry().getConfigServerApiUris(zone), request.getUri()); response.setResponse(applicationView); return response; } private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); if ("container-clustercontroller".equals((serviceName)) && restPath.contains("/status/")) { String result = controller.serviceRegistry().configServer().getClusterControllerStatus(deploymentId, restPath); @@ -1436,7 +1437,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, String restPath, HttpRequest request) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); return controller.serviceRegistry().configServer().getApplicationPackageContent(deploymentId, "/" + restPath, request.getUri()); } @@ -1544,7 +1545,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { /** Schedule reindexing of an application, or a subset of clusters, possibly on a subset of documents. */ private HttpResponse reindex(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); List<String> clusterNames = Optional.ofNullable(request.getProperty("clusterId")).stream() .flatMap(clusters -> Stream.of(clusters.split(","))) .filter(cluster -> ! cluster.isBlank()) @@ -1563,7 +1564,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { /** Gets reindexing status of an application in a zone. */ private HttpResponse getReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); ApplicationReindexing reindexing = controller.applications().applicationReindexing(id, zone); Slime slime = new Slime(); @@ -1620,7 +1621,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { /** Enables reindexing of an application in a zone. */ private HttpResponse enableReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); controller.applications().enableReindexing(id, zone); return new MessageResponse("Enabled reindexing of " + id + " in " + zone); } @@ -1628,7 +1629,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { /** Disables reindexing of an application in a zone. */ private HttpResponse disableReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); controller.applications().disableReindexing(id, zone); return new MessageResponse("Disabled reindexing of " + id + " in " + zone); } @@ -1636,7 +1637,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { /** Schedule restart of deployment, or specific host in a deployment */ private HttpResponse restart(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - ZoneId.from(environment, region)); + requireZone(environment, region)); RestartFilter restartFilter = new RestartFilter() .withHostName(Optional.ofNullable(request.getProperty("hostname")).map(HostName::from)) .withClusterType(Optional.ofNullable(request.getProperty("clusterType")).map(ClusterSpec.Type::from)) @@ -1678,7 +1679,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deploy(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); - ZoneId zone = ZoneId.from(environment, region); + ZoneId zone = requireZone(environment, region); // Get deployOptions Map<String, byte[]> dataParts = parseDataParts(request); @@ -1815,7 +1816,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId id = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - ZoneId.from(environment, region)); + requireZone(environment, region)); // Attempt to deactivate application even if the deployment is not known by the controller controller.applications().deactivate(id.applicationId(), id.zoneId()); return new MessageResponse("Deactivated " + id); @@ -2203,6 +2204,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new MessageResponse("All deployments removed"); } + private ZoneId requireZone(String environment, String region) { + ZoneId zone = ZoneId.from(environment, region); + if (!controller.zoneRegistry().hasZone(zone)) { + throw new IllegalArgumentException("Zone " + zone + " does not exist in this system"); + } + return zone; + } + private static Map<String, byte[]> parseDataParts(HttpRequest request) { String contentHash = request.getHeader("X-Content-Hash"); if (contentHash == null) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e0a4141480d..27d7b6f3d7a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -490,22 +490,22 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("application-clusters.json")); // GET logs - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-central-1/instance/default/logs?from=1233&to=3214", GET) + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-east-1/instance/default/logs?from=1233&to=3214", GET) .userIdentity(USER_ID), "INFO - All good"); // Get content - root - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-central-1/content/", GET).userIdentity(USER_ID), + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/", GET).userIdentity(USER_ID), "{\"path\":\"/\"}"); // Get content - ignore query params - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-central-1/content/bar/file.json?query=param", GET).userIdentity(USER_ID), + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/bar/file.json?query=param", GET).userIdentity(USER_ID), "{\"path\":\"/bar/file.json\"}"); updateMetrics(); // GET metrics - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-central-1/instance/default/metrics", GET) + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-east-1/instance/default/metrics", GET) .userIdentity(USER_ID), new File("proton-metrics.json")); @@ -622,20 +622,20 @@ public class ApplicationApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(HostedAthenzIdentities.from(SCREWDRIVER_ID)); - // POST a 'restart application' in staging environment command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1/restart", POST) + // POST a 'restart application' in staging environment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "{\"message\":\"Requested restart of tenant1.application1.instance1 in staging.us-central-1\"}"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in staging.us-east-3\"}"); - // POST a 'restart application' in staging test command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1/restart", POST) + // POST a 'restart application' in test environment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "{\"message\":\"Requested restart of tenant1.application1.instance1 in test.us-central-1\"}"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in test.us-east-1\"}"); - // POST a 'restart application' in staging dev command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1/restart", POST) + // POST a 'restart application' in dev environment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-central-1\"}"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-east-1\"}"); // POST a 'restart application' command with a host filter (other filters not supported yet) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) @@ -1158,28 +1158,28 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application with an invalid application package MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-failure.json"), 400); // POST (deploy) an application without available capacity configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", "Out of capacity", ConfigServerException.ErrorCode.OUT_OF_CAPACITY, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-out-of-capacity.json"), 400); // POST (deploy) an application where activation fails configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to activate application", "Activation conflict", ConfigServerException.ErrorCode.ACTIVATION_CONFLICT, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-activation-conflict.json"), 409); // POST (deploy) an application where we get an internal server error configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to deploy application", "Internal server error", ConfigServerException.ErrorCode.INTERNAL_SERVER_ERROR, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-internal-server-error.json"), 500); diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index ddea0d4ba85..668798d6c39 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -93,14 +93,10 @@ void BucketId::throwFailedSetUsedBits(uint32_t used, uint32_t availBits) { BucketId::Type BucketId::reverse(Type id) { - Type retVal; - int bytes = sizeof(Type); - - for (int i = 0; i < bytes; i++) { - ((unsigned char*)&retVal)[bytes - i - 1] = reverseBitTable[((const unsigned char*)&id)[i]]; - } - - return retVal; + id = ((id & 0x5555555555555555l) << 1) | ((id & 0xaaaaaaaaaaaaaaaal) >> 1); + id = ((id & 0x3333333333333333l) << 2) | ((id & 0xccccccccccccccccl) >> 2); + id = ((id & 0x0f0f0f0f0f0f0f0fl) << 4) | ((id & 0xf0f0f0f0f0f0f0f0l) >> 4); + return __builtin_bswap64(id); } BucketId::Type diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 6bb39191791..13a3f09449e 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -15,6 +15,7 @@ vespa_add_library(metrics state_api_adapter.cpp summetric.cpp textwriter.cpp + updatehook.cpp valuemetric.cpp valuemetricvalues.cpp xmlwriter.cpp diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 51149cf67c3..8ca74384af0 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -20,7 +20,7 @@ LOG_SETUP(".metrics.manager"); namespace metrics { -typedef MetricsmanagerConfig Config; +using Config = MetricsmanagerConfig; MetricManager::ConsumerSpec::ConsumerSpec() = default; MetricManager::ConsumerSpec::~ConsumerSpec() = default; @@ -32,7 +32,7 @@ MetricManager::Timer::getTime() const { void MetricManager::assertMetricLockLocked(const MetricLockGuard& g) const { - if ((g.mutex() != &_waiter) || !g.owns_lock()) { + if ( ! g.owns(_waiter)) { throw vespalib::IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); } } diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index feedf2c1515..f5ad5c5eea3 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -60,8 +60,6 @@ template class vespalib::hash_set<metrics::Metric::String>; namespace metrics { -using MetricLockGuard = UpdateHook::MetricLockGuard; - class MetricManager : private document::Runnable { public: diff --git a/metrics/src/vespa/metrics/updatehook.cpp b/metrics/src/vespa/metrics/updatehook.cpp new file mode 100644 index 00000000000..b627d55b09e --- /dev/null +++ b/metrics/src/vespa/metrics/updatehook.cpp @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "updatehook.h" + +namespace metrics { + +MetricLockGuard::MetricLockGuard(std::mutex & mutex) + : _guard(mutex) +{} + +bool +MetricLockGuard::owns(const std::mutex & mutex) const { + return (_guard.mutex() == &mutex) && _guard.owns_lock(); +} + +MetricLockGuard::~MetricLockGuard() = default; + +} diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h index 65f34c65664..0a0f218d61a 100644 --- a/metrics/src/vespa/metrics/updatehook.h +++ b/metrics/src/vespa/metrics/updatehook.h @@ -5,6 +5,21 @@ namespace metrics { +class MetricLockGuard { +public: + MetricLockGuard(std::mutex & mutex); + MetricLockGuard(const MetricLockGuard &) = delete; + MetricLockGuard & operator =(const MetricLockGuard &) = delete; + MetricLockGuard(MetricLockGuard &&) = default; + MetricLockGuard & operator =(MetricLockGuard &&) = default; + ~MetricLockGuard(); + + bool owns(const std::mutex &) const; + operator std::unique_lock<std::mutex> & () { return _guard; } +private: + std::unique_lock<std::mutex> _guard; +}; + class MetricManager; class UpdateHook { @@ -14,9 +29,7 @@ class UpdateHook { friend class MetricManager; public: - using UP = std::unique_ptr<UpdateHook>; - using MetricLockGuard = std::unique_lock<std::mutex>; - + using MetricLockGuard = metrics::MetricLockGuard; UpdateHook(const char* name) : _name(name), _nextCall(0), _period(0) {} virtual ~UpdateHook() = default; virtual void updateMetrics(const MetricLockGuard & guard) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 2a7115f4d33..160423c7c68 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -112,6 +112,11 @@ public: vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); } /** + * Returns the underlying executor. Only used for state explorers. + */ + const vespalib::SyncableThreadExecutor& get_executor() const { return _executor; } + + /** * Starts the scheduling thread of this manager. * * @return This, to allow chaining. diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 34aacdafcad..8adc198bb40 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -60,6 +60,11 @@ public: vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); } /** + * Returns the underlying executor. Only used for state explorers. + */ + const vespalib::SyncableThreadExecutor& get_executor() const { return _executor; } + + /** * Closes the request handler interface. This will prevent any more data * from entering this object, allowing you to flush all pending operations * without having to safe-guard against input. diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index 3b1a21a7e2f..7b7fcc9e45d 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -194,7 +194,8 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::DocIdPartition::up } void -DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::update(const MatchingStats &stats) +DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::update(const metrics::MetricLockGuard &, + const MatchingStats &stats) { docsMatched.inc(stats.docsMatched()); docsRanked.inc(stats.docsRanked()); diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index 26dd52a8577..133df81a9e6 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -9,6 +9,8 @@ #include <vespa/metrics/valuemetric.h> #include <vespa/searchcore/proton/matching/matching_stats.h> +namespace metrics { class MetricLockGuard; } + namespace proton { /** @@ -154,7 +156,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet size_t numDocIdPartitions, metrics::MetricSet *parent); ~RankProfileMetrics() override; - void update(const matching::MatchingStats &stats); + void update(const metrics::MetricLockGuard & guard, const matching::MatchingStats &stats); }; using RankProfileMap = std::map<vespalib::string, RankProfileMetrics::UP>; diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 81dd8a64a6c..73b7404ce31 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -35,6 +35,7 @@ vespa_add_library(searchcore_server STATIC documentretrieverbase.cpp documentsubdbcollection.cpp emptysearchview.cpp + executor_explorer_utils.cpp executor_thread_service.cpp executor_threading_service_explorer.cpp executorthreadingservice.cpp @@ -77,6 +78,7 @@ vespa_add_library(searchcore_server STATIC proton_config_snapshot.cpp proton_configurer.cpp proton_disk_layout.cpp + proton_thread_pools_explorer.cpp prune_session_cache_job.cpp pruneremoveddocumentsjob.cpp putdonecontext.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index c7b5aaafbfc..7323744c626 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -35,6 +35,7 @@ #include <vespa/searchlib/common/gatecallback.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/metrics/updatehook.h> #include <vespa/log/log.h> #include <vespa/searchcorespi/index/warmupconfig.h> @@ -102,6 +103,18 @@ findDocumentDB(const ProtonConfig::DocumentdbVector & documentDBs, const vespali return &_G_defaultProtonDocumentDBConfig; } +class MetricsUpdateHook : public metrics::UpdateHook { + DocumentDB &_db; +public: + MetricsUpdateHook(DocumentDB &s) + : metrics::UpdateHook("documentdb-hook"), + _db(s) + {} + void updateMetrics(const MetricLockGuard & guard) override { + _db.updateMetrics(guard); + } +}; + } template <typename FunctionType> @@ -147,7 +160,6 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _configCV(), _activeConfigSnapshot(), _activeConfigSnapshotGeneration(0), - _activeConfigSnapshotSerialNum(0u), _validateAndSanitizeDocStore(protonCfg.validateAndSanitizeDocstore == vespa::config::search::core::ProtonConfig::ValidateAndSanitizeDocstore::YES), _initGate(), _clusterStateHandler(_writeService.master()), @@ -156,7 +168,8 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _config_store(std::move(config_store)), _sessionManager(std::make_shared<matching::SessionManager>(protonCfg.grouping.sessionmanager.maxentries)), _metricsWireService(metricsWireService), - _metricsHook(*this, _docTypeName.getName(), protonCfg.numthreadspersearch), + _metrics(_docTypeName.getName(), protonCfg.numthreadspersearch), + _metricsHook(std::make_unique<MetricsUpdateHook>(*this)), _feedView(), _refCount(), _syncFeedViewEnabled(false), @@ -176,7 +189,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _lidSpaceCompactionHandlers(), _jobTrackers(), _calc(), - _metricsUpdater(_subDBs, _writeService, _jobTrackers, *_sessionManager, _writeFilter, _state) + _metricsUpdater(_subDBs, _writeService, _jobTrackers, *_sessionManager, _writeFilter) { assert(configSnapshot); @@ -222,8 +235,7 @@ void DocumentDB::registerReference() } } -void DocumentDB::setActiveConfig(const DocumentDBConfig::SP &config, - SerialNum serialNum, int64_t generation) { +void DocumentDB::setActiveConfig(const DocumentDBConfig::SP &config, int64_t generation) { lock_guard guard(_configMutex); registerReference(); _activeConfigSnapshot = config; @@ -231,7 +243,6 @@ void DocumentDB::setActiveConfig(const DocumentDBConfig::SP &config, if (_activeConfigSnapshotGeneration < generation) { _activeConfigSnapshotGeneration = generation; } - _activeConfigSnapshotSerialNum = serialNum; _configCV.notify_all(); } @@ -297,7 +308,7 @@ DocumentDB::initFinish(DocumentDBConfig::SP configSnapshot) syncFeedView(); // Check that feed view has been activated. assert(_feedView.get()); - setActiveConfig(configSnapshot, _initConfigSerialNum, configSnapshot->getGeneration()); + setActiveConfig(configSnapshot, configSnapshot->getGeneration()); startTransactionLogReplay(); } @@ -469,7 +480,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum } _state.clearDelayedConfig(); } - setActiveConfig(configSnapshot, serialNum, generation); + setActiveConfig(configSnapshot, generation); if (params.shouldMaintenanceControllerChange()) { forwardMaintenanceConfig(); } @@ -1042,12 +1053,12 @@ DocumentDB::notifyAllBucketsChanged() } void -DocumentDB::updateMetrics(DocumentDBTaggedMetrics &metrics) +DocumentDB::updateMetrics(const metrics::MetricLockGuard & guard) { if (_state.getState() < DDBState::State::REPLAY_TRANSACTION_LOG) { return; } - _metricsUpdater.updateMetrics(metrics); + _metricsUpdater.updateMetrics(guard, _metrics); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index a171861b590..9ea88360b07 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -18,7 +18,6 @@ #include "ireplayconfig.h" #include "maintenancecontroller.h" #include "threading_service_config.h" -#include <vespa/metrics/updatehook.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/monitored_refcount.h> @@ -42,6 +41,10 @@ namespace search { } namespace vespa::config::search::core::internal { class InternalProtonType; } +namespace metrics { + class UpdateHook; + class MetricLockGuard; +} namespace proton { class AttributeConfigInspector; @@ -68,26 +71,6 @@ class DocumentDB : public DocumentDBConfigOwner, public search::transactionlog::SyncProxy { private: - class MetricsUpdateHook : public metrics::UpdateHook { - DocumentDBTaggedMetrics _metrics; - DocumentDB &_db; - public: - MetricsUpdateHook(DocumentDB &s, const std::string &doc_type, size_t maxNumThreads) - : metrics::UpdateHook("documentdb-hook"), - _metrics(doc_type, maxNumThreads), - _db(s) {} - void updateMetrics(const MetricLockGuard & ) override { _db.updateMetrics(_metrics); } - DocumentDBTaggedMetrics &getMetrics() { return _metrics; } - }; - - struct DocumentStoreCacheStats { - search::CacheStats total; - search::CacheStats readySubDb; - search::CacheStats notReadySubDb; - search::CacheStats removedSubDb; - DocumentStoreCacheStats() : total(), readySubDb(), notReadySubDb(), removedSubDb() {} - }; - using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; using IFlushTargetList = std::vector<std::shared_ptr<searchcorespi::IFlushTarget>>; using StatusReportUP = std::unique_ptr<StatusReport>; @@ -114,7 +97,6 @@ private: mutable std::condition_variable _configCV; DocumentDBConfig::SP _activeConfigSnapshot; int64_t _activeConfigSnapshotGeneration; - SerialNum _activeConfigSnapshotSerialNum; const bool _validateAndSanitizeDocStore; vespalib::Gate _initGate; @@ -126,9 +108,10 @@ private: index::IndexConfig _indexCfg; ConfigStore::UP _config_store; std::shared_ptr<matching::SessionManager> _sessionManager; // TODO: This should not have to be a shared pointer. - MetricsWireService &_metricsWireService; - MetricsUpdateHook _metricsHook; - vespalib::VarHolder<IFeedView::SP> _feedView; + MetricsWireService &_metricsWireService; + DocumentDBTaggedMetrics _metrics; + std::unique_ptr<metrics::UpdateHook> _metricsHook; + vespalib::VarHolder<IFeedView::SP> _feedView; MonitoredRefCount _refCount; bool _syncFeedViewEnabled; IDocumentDBOwner &_owner; @@ -145,7 +128,7 @@ private: DocumentDBMetricsUpdater _metricsUpdater; void registerReference(); - void setActiveConfig(const DocumentDBConfig::SP &config, SerialNum serialNum, int64_t generation); + void setActiveConfig(const DocumentDBConfig::SP &config, int64_t generation); DocumentDBConfig::SP getActiveConfig() const; void internalInit(); void initManagers(); @@ -294,7 +277,9 @@ public: * * @return document db metrics **/ - DocumentDBTaggedMetrics &getMetrics() { return _metricsHook.getMetrics(); } + DocumentDBTaggedMetrics &getMetrics() { + return _metrics; + } /** * Obtain the metrics update hook for this document db. @@ -302,7 +287,7 @@ public: * @return metrics update hook **/ metrics::UpdateHook & getMetricsUpdateHook() { - return _metricsHook; + return *_metricsHook; } /** @@ -414,7 +399,7 @@ public: * the metric manager). Do not call this function in multiple * threads at once. **/ - void updateMetrics(DocumentDBTaggedMetrics &metrics); + void updateMetrics(const metrics::MetricLockGuard & guard); /** * Implement search::transactionlog::SyncProxy API. diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp index 3a086046a27..8b923c7a372 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp @@ -34,8 +34,7 @@ DocumentDBMetricsUpdater::DocumentDBMetricsUpdater(const DocumentSubDBCollection ExecutorThreadingService &writeService, DocumentDBJobTrackers &jobTrackers, matching::SessionManager &sessionManager, - const AttributeUsageFilter &writeFilter, - [[maybe_unused]] const DDBState &state) + const AttributeUsageFilter &writeFilter) : _subDBs(subDBs), _writeService(writeService), _jobTrackers(jobTrackers), @@ -170,12 +169,12 @@ updateAttributeMetrics(DocumentDBTaggedMetrics &metrics, const DocumentSubDBColl } void -updateMatchingMetrics(DocumentDBTaggedMetrics &metrics, const IDocumentSubDB &ready) +updateMatchingMetrics(const metrics::MetricLockGuard & guard, DocumentDBTaggedMetrics &metrics, const IDocumentSubDB &ready) { MatchingStats totalStats; for (const auto &rankProfile : metrics.matching.rank_profiles) { MatchingStats matchingStats = ready.getMatcherStats(rankProfile.first); - rankProfile.second->update(matchingStats); + rankProfile.second->update(guard, matchingStats); totalStats.add(matchingStats); } @@ -284,13 +283,13 @@ updateLidSpaceMetrics(MetricSetType &metrics, const search::IDocumentMetaStore & } void -DocumentDBMetricsUpdater::updateMetrics(DocumentDBTaggedMetrics &metrics) +DocumentDBMetricsUpdater::updateMetrics(const metrics::MetricLockGuard & guard, DocumentDBTaggedMetrics &metrics) { TotalStats totalStats; ExecutorThreadingServiceStats threadingServiceStats = _writeService.getStats(); updateIndexMetrics(metrics, _subDBs.getReadySubDB()->getSearchableStats(), totalStats); updateAttributeMetrics(metrics, _subDBs, totalStats); - updateMatchingMetrics(metrics, *_subDBs.getReadySubDB()); + updateMatchingMetrics(guard, metrics, *_subDBs.getReadySubDB()); updateSessionCacheMetrics(metrics, _sessionManager); updateDocumentsMetrics(metrics, _subDBs); updateDocumentStoreMetrics(metrics, _subDBs, _lastDocStoreCacheStats, totalStats); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h index dbf4c45007f..475da7f4e4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h @@ -45,11 +45,10 @@ public: ExecutorThreadingService &writeService, DocumentDBJobTrackers &jobTrackers, matching::SessionManager &sessionManager, - const AttributeUsageFilter &writeFilter, - const DDBState &state); + const AttributeUsageFilter &writeFilter); ~DocumentDBMetricsUpdater(); - void updateMetrics(DocumentDBTaggedMetrics &metrics); + void updateMetrics(const metrics::MetricLockGuard & guard, DocumentDBTaggedMetrics &metrics); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_explorer_utils.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_explorer_utils.cpp new file mode 100644 index 00000000000..bbb87099988 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/executor_explorer_utils.cpp @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "executor_explorer_utils.h" +#include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/singleexecutor.h> +#include <vespa/vespalib/util/threadexecutor.h> +#include <vespa/vespalib/util/threadstackexecutor.h> + +using vespalib::BlockingThreadStackExecutor; +using vespalib::SingleExecutor; +using vespalib::SyncableThreadExecutor; +using vespalib::ThreadStackExecutor; +using vespalib::slime::Cursor; + + +namespace proton::explorer { + +namespace { + +void +convert_syncable_executor_to_slime(const SyncableThreadExecutor& executor, const vespalib::string& type, Cursor& object) +{ + object.setString("type", type); + object.setLong("num_threads", executor.getNumThreads()); + object.setLong("task_limit", executor.getTaskLimit()); +} + +void +convert_single_executor_to_slime(const SingleExecutor& executor, Cursor& object) +{ + convert_syncable_executor_to_slime(executor, "SingleExecutor", object); + object.setLong("watermark", executor.get_watermark()); + object.setDouble("reaction_time_sec", vespalib::to_s(executor.get_reaction_time())); +} + +} + +void +convert_executor_to_slime(const SyncableThreadExecutor* executor, Cursor& object) +{ + if (executor == nullptr) { + return; + } + if (const auto* single = dynamic_cast<const SingleExecutor*>(executor)) { + convert_single_executor_to_slime(*single, object); + } else if (const auto* blocking = dynamic_cast<const BlockingThreadStackExecutor*>(executor)) { + convert_syncable_executor_to_slime(*blocking, "BlockingThreadStackExecutor", object); + } else if (const auto* thread = dynamic_cast<const ThreadStackExecutor*>(executor)) { + convert_syncable_executor_to_slime(*thread, "ThreadStackExecutor", object); + } else { + convert_syncable_executor_to_slime(*executor, "SyncableThreadExecutor", object); + } +} + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_explorer_utils.h b/searchcore/src/vespa/searchcore/proton/server/executor_explorer_utils.h new file mode 100644 index 00000000000..0793fa6ac4a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/executor_explorer_utils.h @@ -0,0 +1,16 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib { class SyncableThreadExecutor; } +namespace vespalib::slime { struct Cursor; } + +namespace proton::explorer { + +/** + * Utility to convert an executor to slime for use with a state explorer. + */ +void convert_executor_to_slime(const vespalib::SyncableThreadExecutor* executor, vespalib::slime::Cursor& object); + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.cpp index 0ecdca54e27..e3154ad6a47 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.cpp @@ -1,26 +1,21 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "executor_explorer_utils.h" #include "executor_threading_service_explorer.h" #include "executorthreadingservice.h" #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/util/adaptive_sequenced_executor.h> -#include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <vespa/vespalib/util/singleexecutor.h> -#include <vespa/vespalib/util/threadexecutor.h> -#include <vespa/vespalib/util/threadstackexecutor.h> using vespalib::AdaptiveSequencedExecutor; -using vespalib::BlockingThreadStackExecutor; using vespalib::ISequencedTaskExecutor; using vespalib::SequencedTaskExecutor; -using vespalib::SingleExecutor; -using vespalib::SyncableThreadExecutor; -using vespalib::ThreadStackExecutor; using vespalib::slime::Cursor; namespace proton { +using explorer::convert_executor_to_slime; + namespace { void @@ -30,39 +25,6 @@ set_type(Cursor& object, const vespalib::string& type) } void -convert_syncable_executor_to_slime(const SyncableThreadExecutor& executor, const vespalib::string& type, Cursor& object) -{ - set_type(object, type); - object.setLong("num_threads", executor.getNumThreads()); - object.setLong("task_limit", executor.getTaskLimit()); -} - -void -convert_single_executor_to_slime(const SingleExecutor& executor, Cursor& object) -{ - convert_syncable_executor_to_slime(executor, "SingleExecutor", object); - object.setLong("watermark", executor.get_watermark()); - object.setDouble("reaction_time_sec", vespalib::to_s(executor.get_reaction_time())); -} - -void -convert_executor_to_slime(const SyncableThreadExecutor* executor, Cursor& object) -{ - if (executor == nullptr) { - return; - } - if (const auto* single = dynamic_cast<const SingleExecutor*>(executor)) { - convert_single_executor_to_slime(*single, object); - } else if (const auto* blocking = dynamic_cast<const BlockingThreadStackExecutor*>(executor)) { - convert_syncable_executor_to_slime(*blocking, "BlockingThreadStackExecutor", object); - } else if (const auto* thread = dynamic_cast<const ThreadStackExecutor*>(executor)) { - convert_syncable_executor_to_slime(*thread, "ThreadStackExecutor", object); - } else { - convert_syncable_executor_to_slime(*executor, "SyncableThreadExecutor", object); - } -} - -void convert_sequenced_executor_to_slime(const SequencedTaskExecutor& executor, Cursor& object) { set_type(object, "SequencedTaskExecutor"); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index dddca6a9ddd..36d630bc519 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -10,6 +10,7 @@ #include "proton.h" #include "proton_config_snapshot.h" #include "proton_disk_layout.h" +#include "proton_thread_pools_explorer.h" #include "resource_usage_explorer.h" #include "searchhandlerproxy.h" #include "simpleflush.h" @@ -34,6 +35,7 @@ #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/random.h> +#include <vespa/metrics/updatehook.h> #include <vespa/searchlib/aggregation/forcelink.hpp> #include <vespa/searchlib/expression/forcelink.hpp> @@ -112,6 +114,18 @@ derive_shared_threads(const ProtonConfig &proton, return std::max(scaledCores, proton.documentdb.size() + proton.flush.maxconcurrent + 1); } +struct MetricsUpdateHook : metrics::UpdateHook +{ + Proton &self; + MetricsUpdateHook(Proton &s) + : metrics::UpdateHook("proton-hook"), + self(s) + {} + void updateMetrics(const MetricLockGuard &guard) override { + self.updateMetrics(guard); + } +}; + const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; VESPA_THREAD_STACK_TAG(proton_shared_executor) @@ -121,7 +135,7 @@ VESPA_THREAD_STACK_TAG(close_executor) } -Proton::ProtonFileHeaderContext::ProtonFileHeaderContext([[maybe_unused]] const Proton &proton_, const vespalib::string &creator) +Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const vespalib::string &creator) : _hostName(), _creator(creator), _cluster(), @@ -186,9 +200,9 @@ Proton::Proton(const config::ConfigUri & configUri, ComponentConfigProducer(), _configUri(configUri), _mutex(), - _metricsHook(*this), + _metricsHook(std::make_unique<MetricsUpdateHook>(*this)), _metricsEngine(std::make_unique<MetricsEngine>()), - _fileHeaderContext(*this, progName), + _fileHeaderContext(progName), _tls(), _diskMemUsageSampler(), _persistenceEngine(), @@ -219,7 +233,6 @@ Proton::Proton(const config::ConfigUri & configUri, _threadPool(128 * 1024), _distributionKey(-1), _isInitializing(true), - _isReplayDone(false), _abortInit(false), _initStarted(false), _initComplete(false), @@ -260,7 +273,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) diskMemUsageSamplerConfig(protonConfig, hwInfo)); _tls = std::make_unique<TLS>(_configUri.createWithNewId(protonConfig.tlsconfigid), _fileHeaderContext); - _metricsEngine->addMetricsHook(_metricsHook); + _metricsEngine->addMetricsHook(*_metricsHook); _fileHeaderContext.setClusterName(protonConfig.clustername, protonConfig.basedir); _matchEngine = std::make_unique<MatchEngine>(protonConfig.numsearcherthreads, protonConfig.numthreadspersearch, @@ -331,7 +344,6 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _executor.sync(); waitForOnlineState(); - _isReplayDone = true; _rpcHooks->set_online(); _flushEngine->start(); @@ -470,7 +482,7 @@ Proton::shutdown_config_fetching_and_state_exposing_components_once() noexcept _customComponentBindToken.reset(); _stateServer.reset(); if (_metricsEngine) { - _metricsEngine->removeMetricsHook(_metricsHook); + _metricsEngine->removeMetricsHook(*_metricsHook); _metricsEngine->stop(); } _has_shut_down_config_and_state_components = true; @@ -714,7 +726,7 @@ updateExecutorMetrics(ExecutorMetrics &metrics, } void -Proton::updateMetrics(const metrics::UpdateHook::MetricLockGuard &) +Proton::updateMetrics(const metrics::MetricLockGuard &) { { ContentProtonMetrics &metrics = _metricsEngine->root(); @@ -836,6 +848,7 @@ const vespalib::string DOCUMENT_DB = "documentdb"; const vespalib::string FLUSH_ENGINE = "flushengine"; const vespalib::string TLS_NAME = "tls"; const vespalib::string RESOURCE_USAGE = "resourceusage"; +const vespalib::string THREAD_POOLS = "threadpools"; struct StateExplorerProxy : vespalib::StateExplorer { const StateExplorer &explorer; @@ -881,8 +894,7 @@ Proton::get_state(const vespalib::slime::Inserter &, bool) const std::vector<vespalib::string> Proton::get_children_names() const { - std::vector<vespalib::string> names({DOCUMENT_DB, MATCH_ENGINE, FLUSH_ENGINE, TLS_NAME, RESOURCE_USAGE}); - return names; + return {DOCUMENT_DB, THREAD_POOLS, MATCH_ENGINE, FLUSH_ENGINE, TLS_NAME, RESOURCE_USAGE}; } std::unique_ptr<vespalib::StateExplorer> @@ -899,6 +911,13 @@ Proton::get_child(vespalib::stringref name) const return std::make_unique<search::transactionlog::TransLogServerExplorer>(_tls->getTransLogServer()); } else if (name == RESOURCE_USAGE && _diskMemUsageSampler) { return std::make_unique<ResourceUsageExplorer>(_diskMemUsageSampler->writeFilter()); + } else if (name == THREAD_POOLS) { + return std::make_unique<ProtonThreadPoolsExplorer>(_sharedExecutor.get(), + (_matchEngine) ? &_matchEngine->get_executor() : nullptr, + (_summaryEngine) ? &_summaryEngine->get_executor() : nullptr, + (_flushEngine) ? &_flushEngine->get_executor() : nullptr, + &_executor, + _warmupExecutor.get()); } return Explorer_UP(nullptr); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index f88cd7bf0cd..2241389a63c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -31,6 +31,7 @@ namespace vespalib { class StateServer; } namespace search::transactionlog { class TransLogServerApp; } +namespace metrics { class MetricLockGuard; } namespace proton { class DiskMemUsageSampler; @@ -60,16 +61,6 @@ private: using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; using BucketSpace = document::BucketSpace; - struct MetricsUpdateHook : metrics::UpdateHook - { - Proton &self; - MetricsUpdateHook(Proton &s) - : metrics::UpdateHook("proton-hook"), - self(s) {} - void updateMetrics(const MetricLockGuard &guard) override { self.updateMetrics(guard); } - }; - friend struct MetricsUpdateHook; - class ProtonFileHeaderContext : public search::common::FileHeaderContext { vespalib::string _hostName; @@ -78,23 +69,23 @@ private: pid_t _pid; public: - ProtonFileHeaderContext(const Proton &proton_, const vespalib::string &creator); + ProtonFileHeaderContext(const vespalib::string &creator); ~ProtonFileHeaderContext() override; void addTags(vespalib::GenericHeader &header, const vespalib::string &name) const override; void setClusterName(const vespalib::string &clusterName, const vespalib::string &baseDir); }; - const config::ConfigUri _configUri; - mutable std::shared_mutex _mutex; - MetricsUpdateHook _metricsHook; - std::unique_ptr<MetricsEngine> _metricsEngine; - ProtonFileHeaderContext _fileHeaderContext; - std::unique_ptr<TLS> _tls; + const config::ConfigUri _configUri; + mutable std::shared_mutex _mutex; + std::unique_ptr<metrics::UpdateHook> _metricsHook; + std::unique_ptr<MetricsEngine> _metricsEngine; + ProtonFileHeaderContext _fileHeaderContext; + std::unique_ptr<TLS> _tls; std::unique_ptr<DiskMemUsageSampler> _diskMemUsageSampler; PersistenceEngine::UP _persistenceEngine; DocumentDBMap _documentDBMap; - std::unique_ptr<MatchEngine> _matchEngine; + std::unique_ptr<MatchEngine> _matchEngine; std::unique_ptr<SummaryEngine> _summaryEngine; std::unique_ptr<DocsumBySlime> _docsumBySlime; MemoryFlushConfigUpdater::UP _memoryFlushConfigUpdater; @@ -138,12 +129,6 @@ private: void applyConfig(const BootstrapConfig::SP & configSnapshot) override; MonitorReply::UP ping(MonitorRequest::UP request, MonitorClient &client) override; - /** - * Called by the metrics update hook (typically in the context of - * the metric manager). Do not call this function in multiple - * threads at once. - **/ - void updateMetrics(const metrics::UpdateHook::MetricLockGuard &guard); void waitForInitDone(); void waitForOnlineState(); uint32_t getDistributionKey() const override { return _distributionKey; } @@ -161,6 +146,13 @@ public: ~Proton() override; /** + * Called by the metrics update hook (typically in the context of + * the metric manager). Do not call this function in multiple + * threads at once. + **/ + void updateMetrics(const metrics::MetricLockGuard &guard); + + /** * This method must be called after the constructor and before the destructor. * If not I will force a 'core' upon you. * All relevant initialization is conducted here. diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp new file mode 100644 index 00000000000..e0db9e29c35 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "executor_explorer_utils.h" +#include "proton_thread_pools_explorer.h" +#include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/util/threadexecutor.h> + +using vespalib::SyncableThreadExecutor; + +namespace proton { + +using explorer::convert_executor_to_slime; + +ProtonThreadPoolsExplorer::ProtonThreadPoolsExplorer(const SyncableThreadExecutor* shared, + const SyncableThreadExecutor* match, + const SyncableThreadExecutor* docsum, + const SyncableThreadExecutor* flush, + const SyncableThreadExecutor* proton, + const SyncableThreadExecutor* warmup) + : _shared(shared), + _match(match), + _docsum(docsum), + _flush(flush), + _proton(proton), + _warmup(warmup) +{ +} + +void +ProtonThreadPoolsExplorer::get_state(const vespalib::slime::Inserter& inserter, bool full) const +{ + auto& object = inserter.insertObject(); + if (full) { + convert_executor_to_slime(_shared, object.setObject("shared")); + convert_executor_to_slime(_match, object.setObject("match")); + convert_executor_to_slime(_docsum, object.setObject("docsum")); + convert_executor_to_slime(_flush, object.setObject("flush")); + convert_executor_to_slime(_proton, object.setObject("proton")); + convert_executor_to_slime(_warmup, object.setObject("warmup")); + } +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h new file mode 100644 index 00000000000..8022a0483c4 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/net/state_explorer.h> + +namespace vespalib { class SyncableThreadExecutor; } + +namespace proton { + +/** + * Class used to explore the shared thread pools used by proton and it's document databases. + */ +class ProtonThreadPoolsExplorer : public vespalib::StateExplorer { +private: + const vespalib::SyncableThreadExecutor* _shared; + const vespalib::SyncableThreadExecutor* _match; + const vespalib::SyncableThreadExecutor* _docsum; + const vespalib::SyncableThreadExecutor* _flush; + const vespalib::SyncableThreadExecutor* _proton; + const vespalib::SyncableThreadExecutor* _warmup; + +public: + ProtonThreadPoolsExplorer(const vespalib::SyncableThreadExecutor* shared, + const vespalib::SyncableThreadExecutor* match, + const vespalib::SyncableThreadExecutor* docsum, + const vespalib::SyncableThreadExecutor* flush, + const vespalib::SyncableThreadExecutor* proton, + const vespalib::SyncableThreadExecutor* warmup); + + void get_state(const vespalib::slime::Inserter& inserter, bool full) const override; +}; + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 7d5d49b19c7..0bd50fc0104 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -120,7 +120,7 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx) _pendingLidsForCommit(std::make_shared<PendingLidTracker>()), _subDbId(cfg._subDbId), _subDbType(cfg._subDbType), - _fileHeaderContext(*this, ctx._fileHeaderContext, _docTypeName, _baseDir), + _fileHeaderContext(ctx._fileHeaderContext, _docTypeName, _baseDir), _gidToLidChangeHandler(std::make_shared<DummyGidToLidChangeHandler>()) { vespalib::mkdir(_baseDir, false); // Assume parent is created. @@ -497,8 +497,7 @@ StoreOnlyDocSubDB::getDocumentDBReference() } StoreOnlySubDBFileHeaderContext:: -StoreOnlySubDBFileHeaderContext([[maybe_unused]] StoreOnlyDocSubDB &owner, - const FileHeaderContext & parentFileHeaderContext, +StoreOnlySubDBFileHeaderContext(const FileHeaderContext & parentFileHeaderContext, const DocTypeName &docTypeName, const vespalib::string &baseDir) : FileHeaderContext(), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 1cdd22fcc41..7c3f7c82eb0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -50,8 +50,6 @@ public: void close() override { } }; -class StoreOnlyDocSubDB; - /** * File header context used by the store-only sub database. * @@ -65,8 +63,7 @@ class StoreOnlySubDBFileHeaderContext : public search::common::FileHeaderContext vespalib::string _subDB; public: - StoreOnlySubDBFileHeaderContext(StoreOnlyDocSubDB &owner, - const search::common::FileHeaderContext & parentFileHeaderContext, + StoreOnlySubDBFileHeaderContext(const search::common::FileHeaderContext & parentFileHeaderContext, const DocTypeName &docTypeName, const vespalib::string &baseDir); ~StoreOnlySubDBFileHeaderContext(); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h index 029aefacfc8..c1cb1f91a2a 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h @@ -65,6 +65,11 @@ public: vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); } /** + * Returns the underlying executor. Only used for state explorers. + */ + const vespalib::SyncableThreadExecutor& get_executor() const { return _executor; } + + /** * Starts the underlying threads. This will throw a vespalib::Exception if * it failed to start for any reason. */ diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp index e9f48421fcf..5a60f2e617f 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp @@ -13,7 +13,7 @@ namespace search::features { TermDistanceExecutor::TermDistanceExecutor(const IQueryEnvironment & env, - [[maybe_unused]] const TermDistanceParams & params) : + const TermDistanceParams & params) : FeatureExecutor(), _termA(env.getTerm(params.termX)), _termB(env.getTerm(params.termY)), diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index 8a508a2169d..89d82df62bd 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -14,6 +14,7 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/metrics/updatehook.h> #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 72115da8de6..928373d516b 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -16,6 +16,7 @@ #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/metrics/updatehook.h> #include <thread> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> @@ -453,7 +454,7 @@ TEST_F(DistributorTest, metric_update_hook_updates_pending_maintenance_metrics) // Force trigger update hook std::mutex l; - distributor_metric_update_hook().updateMetrics(std::unique_lock(l)); + distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l)); // Metrics should now be updated to the last complete working state { const IdealStateMetricSet& metrics(getIdealStateManager().getMetrics()); @@ -482,7 +483,7 @@ TEST_F(DistributorTest, bucket_db_memory_usage_metrics_only_updated_at_fixed_tim tickDistributorNTimes(10); std::mutex l; - distributor_metric_update_hook().updateMetrics(std::unique_lock(l)); + distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l)); auto* m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes"); ASSERT_TRUE(m != nullptr); auto last_used = m->getLongValue("last"); @@ -496,7 +497,7 @@ TEST_F(DistributorTest, bucket_db_memory_usage_metrics_only_updated_at_fixed_tim const auto sample_interval_sec = db_sample_interval_sec(getDistributor()); getClock().setAbsoluteTimeInSeconds(1000 + sample_interval_sec - 1); // Not there yet. tickDistributorNTimes(50); - distributor_metric_update_hook().updateMetrics(std::unique_lock(l)); + distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l)); m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes"); auto now_used = m->getLongValue("last"); @@ -504,7 +505,7 @@ TEST_F(DistributorTest, bucket_db_memory_usage_metrics_only_updated_at_fixed_tim getClock().setAbsoluteTimeInSeconds(1000 + sample_interval_sec + 1); tickDistributorNTimes(10); - distributor_metric_update_hook().updateMetrics(std::unique_lock(l)); + distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l)); m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes"); now_used = m->getLongValue("last"); diff --git a/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h b/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h index fb9606f33e0..6292fe9a598 100644 --- a/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h +++ b/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h @@ -9,10 +9,12 @@ #include <mutex> +namespace metrics { class MetricLockGuard; } + namespace storage::framework { struct MetricUpdateHook { - using MetricLockGuard = std::unique_lock<std::mutex>; + using MetricLockGuard = metrics::MetricLockGuard; virtual ~MetricUpdateHook() = default; virtual void updateMetrics(const MetricLockGuard &) = 0; diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index 316bf389023..f8ca8987a70 100644 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -18,8 +18,8 @@ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implem @Inject public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { + this.peer = new VespaQuorumPeer(); reconfigurer.startOrReconfigure(zookeeperServerConfig, this); - peer = new VespaQuorumPeer(); } @Override diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java index 61951391f89..2eff37a1b4c 100644 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java @@ -1,14 +1,13 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; +import com.yahoo.protect.Process; import org.apache.zookeeper.server.admin.AdminServer; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerMain; import java.io.IOException; import java.nio.file.Path; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Starts/stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps @@ -18,8 +17,6 @@ import java.util.logging.Logger; */ class VespaQuorumPeer extends QuorumPeerMain { - private static final Logger LOG = Logger.getLogger(VespaQuorumPeer.class.getName()); - public void start(Path path) { initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); } @@ -35,8 +32,7 @@ class VespaQuorumPeer extends QuorumPeerMain { // server with the new config, this will fail until the old server is deconstructed. If the old server // fails to deconstruct/shut down, the new one will never start and if that happens forcing a restart is // the better option. - LOG.log(Level.SEVERE, "Failed to shut down properly, forcing restart", e); - System.exit(1); + Process.logAndDie("Failed to shut down ZooKeeper properly, forcing shutdown", e); } } } diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index 316bf389023..f8ca8987a70 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -18,8 +18,8 @@ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implem @Inject public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { + this.peer = new VespaQuorumPeer(); reconfigurer.startOrReconfigure(zookeeperServerConfig, this); - peer = new VespaQuorumPeer(); } @Override diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java index 61951391f89..2eff37a1b4c 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java @@ -1,14 +1,13 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; +import com.yahoo.protect.Process; import org.apache.zookeeper.server.admin.AdminServer; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerMain; import java.io.IOException; import java.nio.file.Path; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Starts/stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps @@ -18,8 +17,6 @@ import java.util.logging.Logger; */ class VespaQuorumPeer extends QuorumPeerMain { - private static final Logger LOG = Logger.getLogger(VespaQuorumPeer.class.getName()); - public void start(Path path) { initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); } @@ -35,8 +32,7 @@ class VespaQuorumPeer extends QuorumPeerMain { // server with the new config, this will fail until the old server is deconstructed. If the old server // fails to deconstruct/shut down, the new one will never start and if that happens forcing a restart is // the better option. - LOG.log(Level.SEVERE, "Failed to shut down properly, forcing restart", e); - System.exit(1); + Process.logAndDie("Failed to shut down ZooKeeper properly, forcing shutdown", e); } } } |