summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorolaaun <olaa@oath.com>2018-10-26 08:55:17 +0200
committerGitHub <noreply@github.com>2018-10-26 08:55:17 +0200
commit57ba28e794416ae7cc6e14660249459dac7df937 (patch)
tree263799b7a75fd7c68f01c44c3d7f7d0615ab8e8a
parent5c771a79843d6ed595c8d419f7ad4ffc42c0fbf7 (diff)
Olaaun/metric maintaier use api (#7402)
* Metric maintainers now feed to API instead of writing directly to DB * Only feed once per maintain run
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java54
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java86
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java35
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java112
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java23
6 files changed, 188 insertions, 126 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
index 135a05e2725..44f29eb6113 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
@@ -3,16 +3,26 @@ package com.yahoo.vespa.hosted.controller.maintenance;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
-import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
import java.time.Duration;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.logging.Level;
/**
* Fetch utilization metrics and update applications with this data.
@@ -22,10 +32,12 @@ import java.util.Map;
public class ClusterUtilizationMaintainer extends Maintainer {
private final Controller controller;
+ private final List<String> baseUris;
- public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) {
+ public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl, ApiAuthorityConfig apiAuthorityConfig) {
super(controller, duration, jobControl);
this.controller = controller;
+ this.baseUris = apiAuthorityConfig.authorities();
}
private Map<ClusterSpec.Id, ClusterUtilization> getUpdatedClusterUtilizations(ApplicationId app, ZoneId zone) {
@@ -43,15 +55,45 @@ public class ClusterUtilizationMaintainer extends Maintainer {
@Override
protected void maintain() {
+ try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) {
+ String uri = baseUris.get(0) + "metricforwarding/v1/clusterutilization"; // For now, we only feed to one controller
+ Slime slime = getMetricSlime();
+ ByteArrayEntity entity = new ByteArrayEntity(SlimeUtils.toJsonBytes(slime));
+ HttpPost httpPost = new HttpPost(uri);
+ httpPost.setEntity(entity);
+ httpClient.execute(httpPost);
+ } catch (Exception e) {
+ log.log(Level.WARNING, "Failed to update cluster utilization metrics", e);
+ }
+
+ }
+
+ private Slime getMetricSlime() {
+ Slime slime = new Slime();
+ Cursor cursor = slime.setArray();
for (Application application : controller().applications().asList()) {
+ Cursor applicationCursor = cursor.addObject();
+ applicationCursor.setString("applicationId", application.id().serializedForm());
+ Cursor deploymentArray = applicationCursor.setArray("deployments");
for (Deployment deployment : application.deployments().values()) {
-
+ Cursor deploymentEntry = deploymentArray.addObject();
+ deploymentEntry.setString("zoneId", deployment.zone().value());
+ Cursor clusterArray = deploymentEntry.setArray("clusterUtil");
Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone());
-
- controller().applications().lockIfPresent(application.id(), lockedApplication ->
- controller().applications().store(lockedApplication.withClusterUtilization(deployment.zone(), clusterUtilization)));
+ fillClusterUtilization(clusterArray, clusterUtilization);
}
}
+ return slime;
}
+ private void fillClusterUtilization(Cursor cursor, Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization) {
+ for (Map.Entry<ClusterSpec.Id, ClusterUtilization> entry : clusterUtilization.entrySet()) {
+ Cursor clusterUtilCursor = cursor.addObject();
+ clusterUtilCursor.setString("clusterSpecId", entry.getKey().value());
+ clusterUtilCursor.setDouble("cpu", entry.getValue().getCpu());
+ clusterUtilCursor.setDouble("memory", entry.getValue().getMemory());
+ clusterUtilCursor.setDouble("disk", entry.getValue().getDisk());
+ clusterUtilCursor.setDouble("diskBusy", entry.getValue().getDiskBusy());
+ }
+ }
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 33555c08a43..5d26ce16885 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -65,8 +65,8 @@ public class ControllerMaintenance extends AbstractComponent {
upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator);
readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofSeconds(30), jobControl);
clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient);
- clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl);
- deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl);
+ clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl, apiAuthorityConfig);
+ deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl, apiAuthorityConfig);
applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues);
dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService);
systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 72936f8a679..67fb224f1ea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -2,15 +2,23 @@
package com.yahoo.vespa.hosted.controller.maintenance;
import com.yahoo.config.provision.HostName;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.ApplicationController;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.RotationStatus;
+import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
import com.yahoo.yolean.Exceptions;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
@@ -37,10 +45,12 @@ public class DeploymentMetricsMaintainer extends Maintainer {
private static final int applicationsToUpdateInParallel = 10;
private final ApplicationController applications;
+ private final List<String> baseUris;
- public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) {
+ public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl, ApiAuthorityConfig apiAuthorityConfig) {
super(controller, duration, jobControl);
this.applications = controller.applications();
+ baseUris = apiAuthorityConfig.authorities();
}
@Override
@@ -51,37 +61,27 @@ public class DeploymentMetricsMaintainer extends Maintainer {
// Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used
ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel);
+ Slime slime = new Slime();
+ Cursor cursor = slime.setArray();
pool.submit(() -> {
applicationList.parallelStream().forEach(application -> {
- try {
- applications.lockIfPresent(application.id(), locked ->
- applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id()))));
-
- applications.lockIfPresent(application.id(), locked ->
- applications.store(locked.withRotationStatus(rotationStatus(application))));
-
- for (Deployment deployment : application.deployments().values()) {
- MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService()
- .getDeploymentMetrics(application.id(), deployment.zone());
- DeploymentMetrics newMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(),
- deploymentMetrics.writesPerSecond(),
- deploymentMetrics.documentCount(),
- deploymentMetrics.queryLatencyMillis(),
- deploymentMetrics.writeLatencyMillis());
-
- applications.lockIfPresent(application.id(), locked ->
- applications.store(locked.with(deployment.zone(), newMetrics)
- .recordActivityAt(controller().clock().instant(), deployment.zone())));
- }
- } catch (Exception e) {
- failures.incrementAndGet();
- lastException.set(e);
+ Cursor applicationCursor = cursor.addObject();
+ applicationCursor.setString("applicationId", application.id().serializedForm());
+ Cursor applicationMetrics = applicationCursor.setObject("applicationMetrics");
+ fillApplicationMetrics(applicationMetrics, application);
+ Cursor rotationStatus = applicationCursor.setArray("rotationStatus");
+ fillRotationStatus(rotationStatus, application);
+ Cursor deploymentArray = applicationCursor.setArray("deploymentMetrics");
+ for (Deployment deployment : application.deployments().values()) {
+ Cursor deploymentEntry = deploymentArray.addObject();
+ fillDeploymentMetrics(deploymentEntry, application, deployment);
}
});
});
pool.shutdown();
try {
pool.awaitTermination(30, TimeUnit.MINUTES);
+ feedMetrics(slime);
if (lastException.get() != null) {
log.log(Level.WARNING, String.format("Failed to query metrics service for %d/%d applications. Last error: %s. Retrying in %s",
failures.get(),
@@ -91,6 +91,8 @@ public class DeploymentMetricsMaintainer extends Maintainer {
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
+ } catch (IOException e) {
+ log.log(Level.WARNING, "Unable to feed metrics to API", e);
}
}
@@ -106,6 +108,40 @@ public class DeploymentMetricsMaintainer extends Maintainer {
.orElseGet(Collections::emptyMap);
}
+ private void fillApplicationMetrics(Cursor applicationCursor, Application application) {
+ MetricsService.ApplicationMetrics metrics = controller().metricsService().getApplicationMetrics(application.id());
+ applicationCursor.setDouble("queryServiceQuality", metrics.queryServiceQuality());
+ applicationCursor.setDouble("writeServiceQuality", metrics.writeServiceQuality());
+ }
+
+ private void fillRotationStatus(Cursor rotationStatusCursor, Application application) {
+ Map<HostName, RotationStatus> rotationStatus = rotationStatus(application);
+ for (Map.Entry<HostName, RotationStatus> entry : rotationStatus.entrySet()) {
+ Cursor rotationStatusEntry = rotationStatusCursor.addObject();
+ rotationStatusEntry.setString("hostname", entry.getKey().value());
+ rotationStatusEntry.setString("rotationStatus", entry.getValue().toString());
+ }
+ }
+
+ private void fillDeploymentMetrics(Cursor deploymentCursor, Application application, Deployment deployment) {
+ MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService()
+ .getDeploymentMetrics(application.id(), deployment.zone());
+ deploymentCursor.setString("zoneId", deployment.zone().value());
+ deploymentCursor.setDouble("queriesPerSecond", deploymentMetrics.queriesPerSecond());
+ deploymentCursor.setDouble("writesPerSecond", deploymentMetrics.writesPerSecond());
+ deploymentCursor.setDouble("documentCount", deploymentMetrics.documentCount());
+ deploymentCursor.setDouble("queryLatencyMillis", deploymentMetrics.queryLatencyMillis());
+ deploymentCursor.setDouble("writeLatencyMillis", deploymentMetrics.writeLatencyMillis());
+ }
+
+ private void feedMetrics(Slime slime) throws IOException {
+ String uri = baseUris.get(0) + "/metricforwarding/v1/deploymentmetrics/"; // For now, we only feed to one controller
+ CloseableHttpClient httpClient = HttpClientBuilder.create().build();
+ HttpPost httpPost = new HttpPost(uri);
+ httpPost.setEntity(new ByteArrayEntity(SlimeUtils.toJsonBytes(slime)));
+ httpClient.execute(httpPost);
+ }
+
private static RotationStatus from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) {
switch (status) {
case IN: return RotationStatus.in;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
index 66be2d09797..71d2690bb4a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java
@@ -3,24 +3,45 @@ package com.yahoo.vespa.hosted.controller.maintenance;
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.github.tomakehurst.wiremock.stubbing.ServeEvent;
+import com.github.tomakehurst.wiremock.verification.LoggedRequest;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
import com.yahoo.vespa.hosted.controller.ControllerTester;
import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
import java.time.Duration;
+import java.util.List;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.findAll;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents;
+import static com.github.tomakehurst.wiremock.client.WireMock.okJson;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static org.junit.Assert.assertEquals;
/**
* @author smorgrav
*/
public class ClusterUtilizationMaintainerTest {
+ @Rule
+ public WireMockRule wireMockRule = new WireMockRule(4443);
+
@Test
public void maintain() {
+ wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/clusterutilization"))
+ .willReturn(aResponse().withStatus(200)));
ControllerTester tester = new ControllerTester();
ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id();
@@ -28,12 +49,16 @@ public class ClusterUtilizationMaintainerTest {
Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
Assert.assertEquals(0, deployment.clusterUtils().size());
- ClusterUtilizationMaintainer mainainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()));
- mainainer.maintain();
+ ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/");
+ ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder);
+ ClusterUtilizationMaintainer maintainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()), apiAuthorityConfig);
+ maintainer.maintain();
- deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
- Assert.assertEquals(1, deployment.clusterUtils().size());
- Assert.assertEquals(0.5554, deployment.clusterUtils().get(ClusterSpec.Id.from("default")).getCpu(), Double.MIN_VALUE);
+ List<ServeEvent> allServeEvents = getAllServeEvents();
+ assertEquals(allServeEvents.size(), 1);
+ LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/clusterutilization"))).get(0);
+ String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"deployments\":[{\"zoneId\":\"dev.us-east-1\",\"clusterUtil\":[{\"clusterSpecId\":\"default\",\"cpu\":0.5554,\"memory\":0.6990000000000001,\"disk\":0.34590000000000004,\"diskBusy\":0.0}]}]}]";
+ assertEquals(expectedBody, new String(request.getBody()));
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index e11440a372c..b0d28e05744 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -1,27 +1,30 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.maintenance;
-import com.yahoo.config.provision.ApplicationId;
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.github.tomakehurst.wiremock.stubbing.ServeEvent;
+import com.github.tomakehurst.wiremock.verification.LoggedRequest;
import com.yahoo.config.provision.Environment;
import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.ControllerTester;
import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.RotationStatus;
+import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock;
+import org.junit.Rule;
import org.junit.Test;
import java.time.Duration;
-import java.time.Instant;
-import java.util.function.Supplier;
-
-import static java.time.temporal.ChronoUnit.MILLIS;
+import java.util.List;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.findAll;
+import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
/**
* @author smorgrav
@@ -29,69 +32,16 @@ import static org.junit.Assert.assertFalse;
*/
public class DeploymentMetricsMaintainerTest {
- @Test
- public void updates_metrics() {
- ControllerTester tester = new ControllerTester();
- ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1",
- Environment.dev, 123).id();
- DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
- Supplier<Application> app = tester.application(appId);
- Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get();
-
- // No metrics gathered yet
- assertEquals(0, app.get().metrics().queryServiceQuality(), 0);
- assertEquals(0, deployment.get().metrics().documentCount(), 0);
- assertFalse("Never received any queries", deployment.get().activity().lastQueried().isPresent());
- assertFalse("Never received any writes", deployment.get().activity().lastWritten().isPresent());
-
- // Metrics are gathered and saved to application
- maintainer.maintain();
- assertEquals(0.5, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE);
- assertEquals(0.7, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE);
- assertEquals(1, deployment.get().metrics().queriesPerSecond(), Double.MIN_VALUE);
- assertEquals(2, deployment.get().metrics().writesPerSecond(), Double.MIN_VALUE);
- assertEquals(3, deployment.get().metrics().documentCount(), Double.MIN_VALUE);
- assertEquals(4, deployment.get().metrics().queryLatencyMillis(), Double.MIN_VALUE);
- assertEquals(5, deployment.get().metrics().writeLatencyMillis(), Double.MIN_VALUE);
- Instant t1 = tester.clock().instant().truncatedTo(MILLIS);
- assertEquals(t1, deployment.get().activity().lastQueried().get());
- assertEquals(t1, deployment.get().activity().lastWritten().get());
-
- // Time passes. Activity is updated as app is still receiving traffic
- tester.clock().advance(Duration.ofHours(1));
- Instant t2 = tester.clock().instant().truncatedTo(MILLIS);
- maintainer.maintain();
- assertEquals(t2, deployment.get().activity().lastQueried().get());
- assertEquals(t2, deployment.get().activity().lastWritten().get());
- assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
- assertEquals(2, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
-
- // Query traffic disappears. Query activity stops updating
- tester.clock().advance(Duration.ofHours(1));
- Instant t3 = tester.clock().instant().truncatedTo(MILLIS);
- tester.metricsService().setMetric("queriesPerSecond", 0D);
- tester.metricsService().setMetric("writesPerSecond", 5D);
- maintainer.maintain();
- assertEquals(t2, deployment.get().activity().lastQueried().get());
- assertEquals(t3, deployment.get().activity().lastWritten().get());
- assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
- assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
-
- // Feed traffic disappears. Feed activity stops updating
- tester.clock().advance(Duration.ofHours(1));
- tester.metricsService().setMetric("writesPerSecond", 0D);
- maintainer.maintain();
- assertEquals(t2, deployment.get().activity().lastQueried().get());
- assertEquals(t3, deployment.get().activity().lastWritten().get());
- assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE);
- assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
- }
+ @Rule
+ public WireMockRule wireMockRule = new WireMockRule(4443);
@Test
- public void updates_rotation_status() {
+ public void maintain() {
DeploymentTester tester = new DeploymentTester();
MetricsServiceMock metricsService = tester.controllerTester().metricsService();
- DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
+ ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/");
+ ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder);
+ DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), apiAuthorityConfig);
Application application = tester.createApplication("app1", "tenant1", 1, 1L);
ZoneId zone1 = ZoneId.from("prod", "us-west-1");
ZoneId zone2 = ZoneId.from("prod", "us-east-3");
@@ -105,32 +55,22 @@ public class DeploymentMetricsMaintainerTest {
.build();
tester.deployCompletely(application, applicationPackage);
- Supplier<Application> app = () -> tester.application(application.id());
- Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1);
- Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2);
String assignedRotation = "rotation-fqdn-01";
tester.controllerTester().metricsService().addRotation(assignedRotation);
- // No status gathered yet
- assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment1.get()));
- assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment2.get()));
-
// One rotation out, one in
metricsService.setZoneIn(assignedRotation, "proxy.prod.us-west-1.vip.test");
metricsService.setZoneOut(assignedRotation,"proxy.prod.us-east-3.vip.test");
- maintainer.maintain();
- assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get()));
- assertEquals(RotationStatus.out, app.get().rotationStatus(deployment2.get()));
- // All rotations in
- metricsService.setZoneIn(assignedRotation,"proxy.prod.us-east-3.vip.test");
+ wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/deploymentmetrics/"))
+ .willReturn(aResponse().withStatus(200)));
maintainer.maintain();
- assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get()));
- assertEquals(RotationStatus.in, app.get().rotationStatus(deployment2.get()));
- }
- private static DeploymentMetricsMaintainer maintainer(Controller controller) {
- return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator()));
+ List<ServeEvent> allServeEvents = getAllServeEvents();
+ assertEquals(1, allServeEvents.size());
+ LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/deploymentmetrics/"))).get(0);
+ String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"applicationMetrics\":{\"queryServiceQuality\":0.5,\"writeServiceQuality\":0.7},\"rotationStatus\":[{\"hostname\":\"proxy.prod.us-east-3.vip.test\",\"rotationStatus\":\"out\"},{\"hostname\":\"proxy.prod.us-west-1.vip.test\",\"rotationStatus\":\"in\"}],\"deploymentMetrics\":[{\"zoneId\":\"prod.us-west-1\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0},{\"zoneId\":\"prod.us-east-3\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0}]}]";
+ assertEquals(expectedBody, new String(request.getBody()));
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 027fc9745f2..f840c188a34 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.AthenzService;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.TenantName;
import com.yahoo.slime.Cursor;
@@ -19,6 +20,7 @@ import com.yahoo.vespa.athenz.api.AthenzUser;
import com.yahoo.vespa.athenz.api.OktaAccessToken;
import com.yahoo.vespa.config.SlimeUtils;
import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
@@ -41,8 +43,10 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock;
import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock;
+import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
@@ -75,6 +79,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.TreeMap;
import java.util.function.Supplier;
import static com.yahoo.application.container.handler.Request.Method.DELETE;
@@ -1273,8 +1278,22 @@ public class ApplicationApiTest extends ControllerContainerTest {
String vipName = "proxy." + zone.value() + ".vip.test";
metricsService().addRotation(rotationName)
.setZoneIn(rotationName, vipName);
-
- new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())).run();
+ ApplicationController applicationController = controllerTester.controller().applications();
+ List<Application> applicationList = applicationController.asList();
+ applicationList.stream().forEach(application -> {
+ applicationController.lockIfPresent(application.id(), locked ->
+ applicationController.store(locked.withRotationStatus(rotationStatus(application))));
+ });}
+
+ private Map<HostName, RotationStatus> rotationStatus(Application application) {
+ return controllerTester.controller().applications().rotationRepository().getRotation(application)
+ .map(rotation -> controllerTester.controller().metricsService().getRotationStatus(rotation.name()))
+ .map(rotationStatus -> {
+ Map<HostName, RotationStatus> result = new TreeMap<>();
+ rotationStatus.forEach((hostname, status) -> result.put(hostname, RotationStatus.in));
+ return result;
+ })
+ .orElseGet(Collections::emptyMap);
}
private void updateContactInformation() {