diff options
Diffstat (limited to 'controller-server/src')
12 files changed, 245 insertions, 22 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 5970494d471..a09dc0589ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -34,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.aws.ApplicationRoles; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -353,15 +354,27 @@ public class ApplicationController { // Carry out deployment without holding the application lock. ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, endpoints, endpointCertificateMetadata, applicationRoles); + // Record the quota usage for this application + var quotaUsage = deploymentQuotaUsage(zone, job.application()); + lockApplicationOrThrow(applicationId, application -> store(application.with(job.application().instance(), instance -> instance.withNewDeployment(zone, revision, platform, clock.instant(), warningsFrom(result), - QuotaUsage.create(result.quotaUsageRate()))))); + quotaUsage)))); return result; } } + private QuotaUsage deploymentQuotaUsage(ZoneId zoneId, ApplicationId applicationId) { + var quotaUsage = configServer.nodeRepository().getApplication(zoneId, applicationId) + .clusters().values().stream() + .map(Cluster::max) + .mapToDouble(max -> max.nodes() * max.nodeResources().cost()) + .sum(); + return QuotaUsage.create(quotaUsage); + } + private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) { return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone) : applicationStore.get(application.tenant(), application.application(), revision)); @@ -429,11 +442,14 @@ public class ApplicationController { ActivateResult result = deploy(instanceId, applicationPackage, zone, platformVersion, endpoints, endpointCertificateMetadata, Optional.empty()); + // Record the quota usage for this application + var quotaUsage = deploymentQuotaUsage(zone, instanceId); + lockApplicationOrThrow(applicationId, application -> store(application.with(instanceId.instance(), instance -> instance.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), warningsFrom(result), - QuotaUsage.create(result.quotaUsageRate()))))); + quotaUsage)))); return result; } } @@ -547,10 +563,8 @@ public class ApplicationController { endpoints, endpointCertificateMetadata, dockerImageRepo, domain, applicationRoles, quota)); - var quotaUsage = configServer.getQuotaUsage(new DeploymentId(application, zone)); - return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), - applicationPackage.zippedContent().length, quotaUsage.rate); + applicationPackage.zippedContent().length); } finally { // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that // any DNS updates can be propagated as early as possible. @@ -567,7 +581,7 @@ public class ApplicationController { PrepareResponse prepareResponse = new PrepareResponse(); prepareResponse.log = List.of(logEntry); prepareResponse.configChangeActions = new ConfigChangeActions(List.of(), List.of()); - return new ActivateResult(new RevisionId("0"), prepareResponse, 0, 0.0); + return new ActivateResult(new RevisionId("0"), prepareResponse, 0); } private LockedApplication withoutDeletedDeployments(LockedApplication application, InstanceName instance) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java index e6c9e52ff69..5379a08afc0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java @@ -13,13 +13,11 @@ public class ActivateResult { private final RevisionId revisionId; private final PrepareResponse prepareResponse; private final long applicationZipSizeBytes; - private final double quotaUsageRate; - public ActivateResult(RevisionId revisionId, PrepareResponse prepareResponse, long applicationZipSizeBytes, double quotaUsageRate) { + public ActivateResult(RevisionId revisionId, PrepareResponse prepareResponse, long applicationZipSizeBytes) { this.revisionId = revisionId; this.prepareResponse = prepareResponse; this.applicationZipSizeBytes = applicationZipSizeBytes; - this.quotaUsageRate = quotaUsageRate; } public long applicationZipSizeBytes() { @@ -33,9 +31,4 @@ public class ActivateResult { public PrepareResponse prepareResponse() { return prepareResponse; } - - public double quotaUsageRate() { - return quotaUsageRate; - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 124b913eb01..b4904ca3cf8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -646,12 +646,10 @@ public class DeploymentStatus { @Override public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return RunList.from(job) - .matching(run -> change.platform().map(run.versions().targetPlatform()::equals).orElse(true)) - .matching(run -> change.application().map(run.versions().targetApplication()::equals).orElse(true)) - .matching(run -> dependent.flatMap(status::deploymentFor) - .map(deployment -> Versions.from(change, deployment)) - .map(run.versions()::targetsMatch) - .orElse(true)) + .matching(run -> run.versions().targetsMatch(Versions.from(change, + status.application, + dependent.flatMap(status::deploymentFor), + status.systemVersion))) .status(RunStatus.success) .asList().stream() .map(run -> run.end().get()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 0e72a1b42a7..6731c30ecd7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -45,6 +45,8 @@ public class ControllerMaintenance extends AbstractComponent { private final ResourceTagMaintainer resourceTagMaintainer; private final SystemRoutingPolicyMaintainer systemRoutingPolicyMaintainer; private final ApplicationMetaDataGarbageCollector applicationMetaDataGarbageCollector; + private final HostRepairMaintainer hostRepairMaintainer; + @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection @@ -75,6 +77,7 @@ public class ControllerMaintenance extends AbstractComponent { resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger()); systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10)); applicationMetaDataGarbageCollector = new ApplicationMetaDataGarbageCollector(controller, Duration.ofHours(12)); + hostRepairMaintainer = new HostRepairMaintainer(controller, Duration.ofHours(12)); } public Upgrader upgrader() { return upgrader; } @@ -102,6 +105,7 @@ public class ControllerMaintenance extends AbstractComponent { rotationStatusUpdater.close(); resourceTagMaintainer.close(); systemRoutingPolicyMaintainer.close(); + hostRepairMaintainer.close(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 7bd2c737fcb..37de7369452 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -25,10 +25,10 @@ public class DeploymentExpirer extends ControllerMaintainer { @Override protected boolean maintain() { boolean success = true; - for (Application application : controller().applications().readable()) + for (Application application : controller().applications().readable()) { for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) { - if ( ! isExpired(deployment)) continue; + if (!isExpired(deployment)) continue; try { log.log(Level.INFO, "Expiring deployment of " + instance.id() + " in " + deployment.zone()); @@ -40,6 +40,7 @@ public class DeploymentExpirer extends ControllerMaintainer { interval()); } } + } return success; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java new file mode 100644 index 00000000000..e3c6862384f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java @@ -0,0 +1,81 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; +import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport; +import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * + * Responsible for keeping track of hosts under repair. + * + * @author olaa + */ +public class HostRepairMaintainer extends ControllerMaintainer { + + private final NodeRepository nodeRepository; + private final HostRepairClient repairClient; + + private static final Logger log = Logger.getLogger(HostRepairMaintainer.class.getName()); + + + public HostRepairMaintainer(Controller controller, Duration interval) { + super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); + this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + this.repairClient = controller.serviceRegistry().hostRepairClient(); + } + + + @Override + protected boolean maintain() { + AtomicInteger exceptions = new AtomicInteger(0); + + controller().zoneRegistry().zones() + .reachable().zones().stream() + .forEach(zoneApi -> { + var nodeTicketMap = nodeRepository.list((zoneApi).getId()) + .stream() + .filter(this::hasOpenTicket) + .collect(Collectors.toMap( + node -> node, + this::getTicketReport) + ); + try { + repairClient.updateRepairStatus(zoneApi, nodeTicketMap); + } catch (Exception e) { + log.warning("Failed to update repair status; " + Exceptions.toMessageString(e)); + exceptions.incrementAndGet(); + } + } + ); + + return exceptions.get() == 0; + } + + + private boolean hasOpenTicket(Node node) { + var reports = node.reports(); + if (!reports.containsKey(RepairTicketReport.getReportId())) { + return false; + } + return "OPEN".equals(getTicketReport(node).getStatus()); + } + + private RepairTicketReport getTicketReport(Node node) { + return uncheck(() -> RepairTicketReport.fromJsonNode(node.reports().get(RepairTicketReport.getReportId()))); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index f1306b51b39..6e5a9ddc7ab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -24,6 +24,7 @@ import java.time.Instant; import java.util.Collection; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.OptionalLong; import java.util.stream.Collectors; @@ -1216,4 +1217,42 @@ public class DeploymentTriggerTest { app.assertNotRunning(stagingTest); } + @Test + public void testTriggeringOfIdleTestJobsWhenFirstDeploymentIsOnNewerVersionThanChange() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder().systemTest() + .stagingTest() + .region("us-east-3") + .region("us-west-1") + .build(); + var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); + var appToAvoidVersionGC = tester.newDeploymentContext("g", "c", "default").submit().deploy(); + + Version version2 = new Version("7.8.9"); + Version version3 = new Version("8.9.10"); + tester.controllerTester().upgradeSystem(version2); + tester.deploymentTrigger().triggerChange(appToAvoidVersionGC.instanceId(), Change.of(version2)); + appToAvoidVersionGC.deployPlatform(version2); + + // app upgrades first zone to version3, and then the other two to version2. + tester.controllerTester().upgradeSystem(version3); + tester.deploymentTrigger().triggerChange(app.instanceId(), Change.of(version3)); + app.runJob(systemTest).runJob(stagingTest); + tester.triggerJobs(); + tester.upgrader().overrideConfidence(version3, VespaVersion.Confidence.broken); + tester.controllerTester().computeVersionStatus(); + tester.upgrader().run(); + assertEquals(Optional.of(version2), app.instance().change().platform()); + + app.runJob(systemTest) + .runJob(productionUsEast3) + .runJob(stagingTest) + .runJob(productionUsWest1); + + assertEquals(version3, app.instanceJobs().get(productionUsEast3).lastSuccess().get().versions().targetPlatform()); + assertEquals(version2, app.instanceJobs().get(productionUsWest1).lastSuccess().get().versions().targetPlatform()); + assertEquals(Map.of(), app.deploymentStatus().jobsToRun()); + assertEquals(Change.empty(), app.instance().change()); + assertEquals(List.of(), tester.jobs().active()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java new file mode 100644 index 00000000000..d2901aeac97 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.zone.ZoneId; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author ogronnesby + */ +public class QuotaUsageTest { + + @Test + public void testQuotaUsageIsPersisted() { + var tester = new DeploymentTester(); + var context = tester.newDeploymentContext().submit().deploy(); + assertEquals(1.062, context.deployment(ZoneId.from("prod.us-west-1")).quota().rate(), 0.01); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 90276b6b590..72cc000ef98 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; +import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -226,6 +227,11 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.get(zoneId).remove(HostName.from(hostName)); } + @Override + public void patchNode(ZoneId zoneId, String hostName, NodeRepositoryNode node) { + throw new UnsupportedOperationException(); + } + public Optional<Duration> osUpgradeBudget(ZoneId zone, NodeType type, Version version) { return Optional.ofNullable(osUpgradeBudgets.get(Objects.hash(zone, type, version))); } @@ -264,4 +270,8 @@ public class NodeRepositoryMock implements NodeRepository { modifyNodes(deployment, hostname, node -> new Node.Builder(node).rebootGeneration(node.rebootGeneration() + 1).build()); } + public void addReport(ZoneId zoneId, HostName hostName, String reportId, JsonNode report) { + nodeRepository.get(zoneId).get(hostName).reports().put(reportId, report); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 1b21f7db7c4..3ec02c6ceb7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -20,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; +import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient; +import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; @@ -61,6 +63,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); private final ApplicationRoleService applicationRoleService = new NoopApplicationRoleService(); private final BillingController billingController = new MockBillingController(); + private final MockRepairClient repairClient = new MockRepairClient(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -192,6 +195,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return billingController; } + @Override + public MockRepairClient hostRepairClient() { + return repairClient; + } + public ConfigServerMock configServerMock() { return configServerMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java new file mode 100644 index 00000000000..556755581fe --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java @@ -0,0 +1,51 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient; +import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient; +import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class HostRepairMaintainerTest { + + private final ControllerTester tester = new ControllerTester(); + private final HostRepairMaintainer maintainer = new HostRepairMaintainer(tester.controller(), Duration.ofHours(12)); + + @Test + public void maintain() { + var zoneId = ZoneId.from("dev.us-east-1"); + var hostname1 = HostName.from("node-1-tenant-host-dev.us-east-1"); + var hostname2 = HostName.from("node-2-tenant-host-dev.us-east-1"); + var timestamp = Instant.now().toEpochMilli(); + var openTicket = new RepairTicketReport("OPEN", "ticket-1", timestamp, timestamp); + var closedTicket = new RepairTicketReport("CLOSED", "ticket-2", timestamp, timestamp); + + tester.configServer().nodeRepository().addReport( + zoneId, + hostname1, + RepairTicketReport.getReportId(), + openTicket.toJsonNode()); + tester.configServer().nodeRepository().addReport( + zoneId, + hostname2, + RepairTicketReport.getReportId(), + closedTicket.toJsonNode()); + + maintainer.maintain(); + var updatedNodes = tester.serviceRegistry().hostRepairClient().getUpdatedNodes(); + assertEquals(1, updatedNodes.size()); + assertEquals(hostname1, updatedNodes.get(0).hostname()); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 385f0fbc3cf..bb3578b2482 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -28,6 +28,9 @@ "name": "DeploymentMetricsMaintainer" }, { + "name": "HostRepairMaintainer" + }, + { "name": "JobRunner" }, { |