diff options
author | olaaun <olaa@oath.com> | 2018-10-26 10:22:00 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-26 10:22:00 +0200 |
commit | 3e1c1bbf27ece0cc3846df31f13bdedf68453f18 (patch) | |
tree | 55362a2142020d02b591ba2770d28714a819036e /controller-server | |
parent | b04d5cf8899eefa65cbc0112404e72285959cba8 (diff) |
Revert "Olaaun/metric maintaier use api (#7402)" (#7461)
This reverts commit 57ba28e794416ae7cc6e14660249459dac7df937.
Diffstat (limited to 'controller-server')
6 files changed, 126 insertions, 188 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index 44f29eb6113..135a05e2725 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,26 +3,16 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; import java.time.Duration; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.logging.Level; /** * Fetch utilization metrics and update applications with this data. @@ -32,12 +22,10 @@ import java.util.logging.Level; public class ClusterUtilizationMaintainer extends Maintainer { private final Controller controller; - private final List<String> baseUris; - public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl, ApiAuthorityConfig apiAuthorityConfig) { + public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) { super(controller, duration, jobControl); this.controller = controller; - this.baseUris = apiAuthorityConfig.authorities(); } private Map<ClusterSpec.Id, ClusterUtilization> getUpdatedClusterUtilizations(ApplicationId app, ZoneId zone) { @@ -55,45 +43,15 @@ public class ClusterUtilizationMaintainer extends Maintainer { @Override protected void maintain() { - try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { - String uri = baseUris.get(0) + "metricforwarding/v1/clusterutilization"; // For now, we only feed to one controller - Slime slime = getMetricSlime(); - ByteArrayEntity entity = new ByteArrayEntity(SlimeUtils.toJsonBytes(slime)); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(entity); - httpClient.execute(httpPost); - } catch (Exception e) { - log.log(Level.WARNING, "Failed to update cluster utilization metrics", e); - } - - } - - private Slime getMetricSlime() { - Slime slime = new Slime(); - Cursor cursor = slime.setArray(); for (Application application : controller().applications().asList()) { - Cursor applicationCursor = cursor.addObject(); - applicationCursor.setString("applicationId", application.id().serializedForm()); - Cursor deploymentArray = applicationCursor.setArray("deployments"); for (Deployment deployment : application.deployments().values()) { - Cursor deploymentEntry = deploymentArray.addObject(); - deploymentEntry.setString("zoneId", deployment.zone().value()); - Cursor clusterArray = deploymentEntry.setArray("clusterUtil"); + Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone()); - fillClusterUtilization(clusterArray, clusterUtilization); + + controller().applications().lockIfPresent(application.id(), lockedApplication -> + controller().applications().store(lockedApplication.withClusterUtilization(deployment.zone(), clusterUtilization))); } } - return slime; } - private void fillClusterUtilization(Cursor cursor, Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization) { - for (Map.Entry<ClusterSpec.Id, ClusterUtilization> entry : clusterUtilization.entrySet()) { - Cursor clusterUtilCursor = cursor.addObject(); - clusterUtilCursor.setString("clusterSpecId", entry.getKey().value()); - clusterUtilCursor.setDouble("cpu", entry.getValue().getCpu()); - clusterUtilCursor.setDouble("memory", entry.getValue().getMemory()); - clusterUtilCursor.setDouble("disk", entry.getValue().getDisk()); - clusterUtilCursor.setDouble("diskBusy", entry.getValue().getDiskBusy()); - } - } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 5d26ce16885..33555c08a43 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -65,8 +65,8 @@ public class ControllerMaintenance extends AbstractComponent { upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofSeconds(30), jobControl); clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient); - clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl, apiAuthorityConfig); - deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl, apiAuthorityConfig); + clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); + deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 67fb224f1ea..72936f8a679 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -2,23 +2,15 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.yolean.Exceptions; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import java.io.IOException; import java.time.Duration; import java.util.Collections; import java.util.List; @@ -45,12 +37,10 @@ public class DeploymentMetricsMaintainer extends Maintainer { private static final int applicationsToUpdateInParallel = 10; private final ApplicationController applications; - private final List<String> baseUris; - public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl, ApiAuthorityConfig apiAuthorityConfig) { + public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) { super(controller, duration, jobControl); this.applications = controller.applications(); - baseUris = apiAuthorityConfig.authorities(); } @Override @@ -61,27 +51,37 @@ public class DeploymentMetricsMaintainer extends Maintainer { // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel); - Slime slime = new Slime(); - Cursor cursor = slime.setArray(); pool.submit(() -> { applicationList.parallelStream().forEach(application -> { - Cursor applicationCursor = cursor.addObject(); - applicationCursor.setString("applicationId", application.id().serializedForm()); - Cursor applicationMetrics = applicationCursor.setObject("applicationMetrics"); - fillApplicationMetrics(applicationMetrics, application); - Cursor rotationStatus = applicationCursor.setArray("rotationStatus"); - fillRotationStatus(rotationStatus, application); - Cursor deploymentArray = applicationCursor.setArray("deploymentMetrics"); - for (Deployment deployment : application.deployments().values()) { - Cursor deploymentEntry = deploymentArray.addObject(); - fillDeploymentMetrics(deploymentEntry, application, deployment); + try { + applications.lockIfPresent(application.id(), locked -> + applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id())))); + + applications.lockIfPresent(application.id(), locked -> + applications.store(locked.withRotationStatus(rotationStatus(application)))); + + for (Deployment deployment : application.deployments().values()) { + MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService() + .getDeploymentMetrics(application.id(), deployment.zone()); + DeploymentMetrics newMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(), + deploymentMetrics.writesPerSecond(), + deploymentMetrics.documentCount(), + deploymentMetrics.queryLatencyMillis(), + deploymentMetrics.writeLatencyMillis()); + + applications.lockIfPresent(application.id(), locked -> + applications.store(locked.with(deployment.zone(), newMetrics) + .recordActivityAt(controller().clock().instant(), deployment.zone()))); + } + } catch (Exception e) { + failures.incrementAndGet(); + lastException.set(e); } }); }); pool.shutdown(); try { pool.awaitTermination(30, TimeUnit.MINUTES); - feedMetrics(slime); if (lastException.get() != null) { log.log(Level.WARNING, String.format("Failed to query metrics service for %d/%d applications. Last error: %s. Retrying in %s", failures.get(), @@ -91,8 +91,6 @@ public class DeploymentMetricsMaintainer extends Maintainer { } } catch (InterruptedException e) { throw new RuntimeException(e); - } catch (IOException e) { - log.log(Level.WARNING, "Unable to feed metrics to API", e); } } @@ -108,40 +106,6 @@ public class DeploymentMetricsMaintainer extends Maintainer { .orElseGet(Collections::emptyMap); } - private void fillApplicationMetrics(Cursor applicationCursor, Application application) { - MetricsService.ApplicationMetrics metrics = controller().metricsService().getApplicationMetrics(application.id()); - applicationCursor.setDouble("queryServiceQuality", metrics.queryServiceQuality()); - applicationCursor.setDouble("writeServiceQuality", metrics.writeServiceQuality()); - } - - private void fillRotationStatus(Cursor rotationStatusCursor, Application application) { - Map<HostName, RotationStatus> rotationStatus = rotationStatus(application); - for (Map.Entry<HostName, RotationStatus> entry : rotationStatus.entrySet()) { - Cursor rotationStatusEntry = rotationStatusCursor.addObject(); - rotationStatusEntry.setString("hostname", entry.getKey().value()); - rotationStatusEntry.setString("rotationStatus", entry.getValue().toString()); - } - } - - private void fillDeploymentMetrics(Cursor deploymentCursor, Application application, Deployment deployment) { - MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService() - .getDeploymentMetrics(application.id(), deployment.zone()); - deploymentCursor.setString("zoneId", deployment.zone().value()); - deploymentCursor.setDouble("queriesPerSecond", deploymentMetrics.queriesPerSecond()); - deploymentCursor.setDouble("writesPerSecond", deploymentMetrics.writesPerSecond()); - deploymentCursor.setDouble("documentCount", deploymentMetrics.documentCount()); - deploymentCursor.setDouble("queryLatencyMillis", deploymentMetrics.queryLatencyMillis()); - deploymentCursor.setDouble("writeLatencyMillis", deploymentMetrics.writeLatencyMillis()); - } - - private void feedMetrics(Slime slime) throws IOException { - String uri = baseUris.get(0) + "/metricforwarding/v1/deploymentmetrics/"; // For now, we only feed to one controller - CloseableHttpClient httpClient = HttpClientBuilder.create().build(); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new ByteArrayEntity(SlimeUtils.toJsonBytes(slime))); - httpClient.execute(httpPost); - } - private static RotationStatus from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) { switch (status) { case IN: return RotationStatus.in; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java index 71d2690bb4a..66be2d09797 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java @@ -3,45 +3,24 @@ package com.yahoo.vespa.hosted.controller.maintenance; // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.github.tomakehurst.wiremock.stubbing.ServeEvent; -import com.github.tomakehurst.wiremock.verification.LoggedRequest; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Assert; -import org.junit.Rule; import org.junit.Test; import java.time.Duration; -import java.util.List; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.findAll; -import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents; -import static com.github.tomakehurst.wiremock.client.WireMock.okJson; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static org.junit.Assert.assertEquals; /** * @author smorgrav */ public class ClusterUtilizationMaintainerTest { - @Rule - public WireMockRule wireMockRule = new WireMockRule(4443); - @Test public void maintain() { - wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/clusterutilization")) - .willReturn(aResponse().withStatus(200))); ControllerTester tester = new ControllerTester(); ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); @@ -49,16 +28,12 @@ public class ClusterUtilizationMaintainerTest { Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); Assert.assertEquals(0, deployment.clusterUtils().size()); - ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/"); - ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder); - ClusterUtilizationMaintainer maintainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()), apiAuthorityConfig); - maintainer.maintain(); + ClusterUtilizationMaintainer mainainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb())); + mainainer.maintain(); - List<ServeEvent> allServeEvents = getAllServeEvents(); - assertEquals(allServeEvents.size(), 1); - LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/clusterutilization"))).get(0); - String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"deployments\":[{\"zoneId\":\"dev.us-east-1\",\"clusterUtil\":[{\"clusterSpecId\":\"default\",\"cpu\":0.5554,\"memory\":0.6990000000000001,\"disk\":0.34590000000000004,\"diskBusy\":0.0}]}]}]"; - assertEquals(expectedBody, new String(request.getBody())); + deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); + Assert.assertEquals(1, deployment.clusterUtils().size()); + Assert.assertEquals(0.5554, deployment.clusterUtils().get(ClusterSpec.Id.from("default")).getCpu(), Double.MIN_VALUE); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index b0d28e05744..e11440a372c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -1,30 +1,27 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.github.tomakehurst.wiremock.stubbing.ServeEvent; -import com.github.tomakehurst.wiremock.verification.LoggedRequest; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock; -import org.junit.Rule; import org.junit.Test; import java.time.Duration; -import java.util.List; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.findAll; -import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import java.time.Instant; +import java.util.function.Supplier; + +import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * @author smorgrav @@ -32,16 +29,69 @@ import static org.junit.Assert.assertEquals; */ public class DeploymentMetricsMaintainerTest { - @Rule - public WireMockRule wireMockRule = new WireMockRule(4443); + @Test + public void updates_metrics() { + ControllerTester tester = new ControllerTester(); + ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1", + Environment.dev, 123).id(); + DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); + Supplier<Application> app = tester.application(appId); + Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get(); + + // No metrics gathered yet + assertEquals(0, app.get().metrics().queryServiceQuality(), 0); + assertEquals(0, deployment.get().metrics().documentCount(), 0); + assertFalse("Never received any queries", deployment.get().activity().lastQueried().isPresent()); + assertFalse("Never received any writes", deployment.get().activity().lastWritten().isPresent()); + + // Metrics are gathered and saved to application + maintainer.maintain(); + assertEquals(0.5, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE); + assertEquals(0.7, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE); + assertEquals(1, deployment.get().metrics().queriesPerSecond(), Double.MIN_VALUE); + assertEquals(2, deployment.get().metrics().writesPerSecond(), Double.MIN_VALUE); + assertEquals(3, deployment.get().metrics().documentCount(), Double.MIN_VALUE); + assertEquals(4, deployment.get().metrics().queryLatencyMillis(), Double.MIN_VALUE); + assertEquals(5, deployment.get().metrics().writeLatencyMillis(), Double.MIN_VALUE); + Instant t1 = tester.clock().instant().truncatedTo(MILLIS); + assertEquals(t1, deployment.get().activity().lastQueried().get()); + assertEquals(t1, deployment.get().activity().lastWritten().get()); + + // Time passes. Activity is updated as app is still receiving traffic + tester.clock().advance(Duration.ofHours(1)); + Instant t2 = tester.clock().instant().truncatedTo(MILLIS); + maintainer.maintain(); + assertEquals(t2, deployment.get().activity().lastQueried().get()); + assertEquals(t2, deployment.get().activity().lastWritten().get()); + assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE); + assertEquals(2, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE); + + // Query traffic disappears. Query activity stops updating + tester.clock().advance(Duration.ofHours(1)); + Instant t3 = tester.clock().instant().truncatedTo(MILLIS); + tester.metricsService().setMetric("queriesPerSecond", 0D); + tester.metricsService().setMetric("writesPerSecond", 5D); + maintainer.maintain(); + assertEquals(t2, deployment.get().activity().lastQueried().get()); + assertEquals(t3, deployment.get().activity().lastWritten().get()); + assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE); + assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE); + + // Feed traffic disappears. Feed activity stops updating + tester.clock().advance(Duration.ofHours(1)); + tester.metricsService().setMetric("writesPerSecond", 0D); + maintainer.maintain(); + assertEquals(t2, deployment.get().activity().lastQueried().get()); + assertEquals(t3, deployment.get().activity().lastWritten().get()); + assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE); + assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE); + } @Test - public void maintain() { + public void updates_rotation_status() { DeploymentTester tester = new DeploymentTester(); MetricsServiceMock metricsService = tester.controllerTester().metricsService(); - ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/"); - ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder); - DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), apiAuthorityConfig); + DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); Application application = tester.createApplication("app1", "tenant1", 1, 1L); ZoneId zone1 = ZoneId.from("prod", "us-west-1"); ZoneId zone2 = ZoneId.from("prod", "us-east-3"); @@ -55,22 +105,32 @@ public class DeploymentMetricsMaintainerTest { .build(); tester.deployCompletely(application, applicationPackage); + Supplier<Application> app = () -> tester.application(application.id()); + Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1); + Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2); String assignedRotation = "rotation-fqdn-01"; tester.controllerTester().metricsService().addRotation(assignedRotation); + // No status gathered yet + assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment1.get())); + assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment2.get())); + // One rotation out, one in metricsService.setZoneIn(assignedRotation, "proxy.prod.us-west-1.vip.test"); metricsService.setZoneOut(assignedRotation,"proxy.prod.us-east-3.vip.test"); + maintainer.maintain(); + assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get())); + assertEquals(RotationStatus.out, app.get().rotationStatus(deployment2.get())); - wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/deploymentmetrics/")) - .willReturn(aResponse().withStatus(200))); + // All rotations in + metricsService.setZoneIn(assignedRotation,"proxy.prod.us-east-3.vip.test"); maintainer.maintain(); + assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get())); + assertEquals(RotationStatus.in, app.get().rotationStatus(deployment2.get())); + } - List<ServeEvent> allServeEvents = getAllServeEvents(); - assertEquals(1, allServeEvents.size()); - LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/deploymentmetrics/"))).get(0); - String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"applicationMetrics\":{\"queryServiceQuality\":0.5,\"writeServiceQuality\":0.7},\"rotationStatus\":[{\"hostname\":\"proxy.prod.us-east-3.vip.test\",\"rotationStatus\":\"out\"},{\"hostname\":\"proxy.prod.us-west-1.vip.test\",\"rotationStatus\":\"in\"}],\"deploymentMetrics\":[{\"zoneId\":\"prod.us-west-1\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0},{\"zoneId\":\"prod.us-east-3\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0}]}]"; - assertEquals(expectedBody, new String(request.getBody())); + private static DeploymentMetricsMaintainer maintainer(Controller controller) { + return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator())); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index f840c188a34..027fc9745f2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Cursor; @@ -20,7 +19,6 @@ import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; @@ -43,10 +41,8 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; @@ -79,7 +75,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.TreeMap; import java.util.function.Supplier; import static com.yahoo.application.container.handler.Request.Method.DELETE; @@ -1278,22 +1273,8 @@ public class ApplicationApiTest extends ControllerContainerTest { String vipName = "proxy." + zone.value() + ".vip.test"; metricsService().addRotation(rotationName) .setZoneIn(rotationName, vipName); - ApplicationController applicationController = controllerTester.controller().applications(); - List<Application> applicationList = applicationController.asList(); - applicationList.stream().forEach(application -> { - applicationController.lockIfPresent(application.id(), locked -> - applicationController.store(locked.withRotationStatus(rotationStatus(application)))); - });} - - private Map<HostName, RotationStatus> rotationStatus(Application application) { - return controllerTester.controller().applications().rotationRepository().getRotation(application) - .map(rotation -> controllerTester.controller().metricsService().getRotationStatus(rotation.name())) - .map(rotationStatus -> { - Map<HostName, RotationStatus> result = new TreeMap<>(); - rotationStatus.forEach((hostname, status) -> result.put(hostname, RotationStatus.in)); - return result; - }) - .orElseGet(Collections::emptyMap); + + new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())).run(); } private void updateContactInformation() { |