Merge pull request #15019 from vespa-engine/olaa/host-repair-maintenance

Create maintainer tracking host repair status

6 files changed, 157 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 0e72a1b42a7..6731c30ecd7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -45,6 +45,8 @@ public class ControllerMaintenance extends AbstractComponent {
     private final ResourceTagMaintainer resourceTagMaintainer;
     private final SystemRoutingPolicyMaintainer systemRoutingPolicyMaintainer;
     private final ApplicationMetaDataGarbageCollector applicationMetaDataGarbageCollector;
+    private final HostRepairMaintainer hostRepairMaintainer;
+
 
     @Inject
     @SuppressWarnings("unused") // instantiated by Dependency Injection
@@ -75,6 +77,7 @@ public class ControllerMaintenance extends AbstractComponent {
         resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger());
         systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10));
         applicationMetaDataGarbageCollector = new ApplicationMetaDataGarbageCollector(controller, Duration.ofHours(12));
+        hostRepairMaintainer = new HostRepairMaintainer(controller, Duration.ofHours(12));
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -102,6 +105,7 @@ public class ControllerMaintenance extends AbstractComponent {
         rotationStatusUpdater.close();
         resourceTagMaintainer.close();
         systemRoutingPolicyMaintainer.close();
+        hostRepairMaintainer.close();
     }
 
     /** Create one OS upgrader per cloud found in the zone registry of controller */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java
new file mode 100644
index 00000000000..e3c6862384f
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java
@@ -0,0 +1,81 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ *
+ * Responsible for keeping track of hosts under repair.
+ *
+ * @author olaa
+ */
+public class HostRepairMaintainer extends ControllerMaintainer {
+
+    private final NodeRepository nodeRepository;
+    private final HostRepairClient repairClient;
+
+    private static final Logger log = Logger.getLogger(HostRepairMaintainer.class.getName());
+
+
+    public HostRepairMaintainer(Controller controller, Duration interval) {
+        super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)));
+        this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
+        this.repairClient = controller.serviceRegistry().hostRepairClient();
+    }
+
+
+    @Override
+    protected boolean maintain() {
+        AtomicInteger exceptions = new AtomicInteger(0);
+
+        controller().zoneRegistry().zones()
+                .reachable().zones().stream()
+                .forEach(zoneApi -> {
+                            var nodeTicketMap = nodeRepository.list((zoneApi).getId())
+                                    .stream()
+                                    .filter(this::hasOpenTicket)
+                                    .collect(Collectors.toMap(
+                                            node -> node,
+                                            this::getTicketReport)
+                                    );
+                            try {
+                                repairClient.updateRepairStatus(zoneApi, nodeTicketMap);
+                            } catch (Exception e) {
+                                log.warning("Failed to update repair status; " + Exceptions.toMessageString(e));
+                                exceptions.incrementAndGet();
+                            }
+                        }
+                );
+
+        return exceptions.get() == 0;
+    }
+
+
+    private boolean hasOpenTicket(Node node) {
+        var reports = node.reports();
+        if (!reports.containsKey(RepairTicketReport.getReportId())) {
+            return false;
+        }
+        return "OPEN".equals(getTicketReport(node).getStatus());
+    }
+
+    private RepairTicketReport getTicketReport(Node node) {
+        return uncheck(() -> RepairTicketReport.fromJsonNode(node.reports().get(RepairTicketReport.getReportId())));
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
index 90276b6b590..72cc000ef98 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
@@ -226,6 +227,11 @@ public class NodeRepositoryMock implements NodeRepository {
         nodeRepository.get(zoneId).remove(HostName.from(hostName));
     }
 
+    @Override
+    public void patchNode(ZoneId zoneId, String hostName, NodeRepositoryNode node) {
+        throw new UnsupportedOperationException();
+    }
+
     public Optional<Duration> osUpgradeBudget(ZoneId zone, NodeType type, Version version) {
         return Optional.ofNullable(osUpgradeBudgets.get(Objects.hash(zone, type, version)));
     }
@@ -264,4 +270,8 @@ public class NodeRepositoryMock implements NodeRepository {
         modifyNodes(deployment, hostname, node -> new Node.Builder(node).rebootGeneration(node.rebootGeneration() + 1).build());
     }
 
+    public void addReport(ZoneId zoneId, HostName hostName, String reportId, JsonNode report) {
+        nodeRepository.get(zoneId).get(hostName).reports().put(reportId, report);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index 1b21f7db7c4..3ec02c6ceb7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -20,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
 import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService;
@@ -61,6 +63,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final MockResourceTagger mockResourceTagger = new MockResourceTagger();
     private final ApplicationRoleService applicationRoleService = new NoopApplicationRoleService();
     private final BillingController billingController = new MockBillingController();
+    private final MockRepairClient repairClient = new MockRepairClient();
 
     public ServiceRegistryMock(SystemName system) {
         this.zoneRegistryMock = new ZoneRegistryMock(system);
@@ -192,6 +195,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
         return billingController;
     }
 
+    @Override
+    public MockRepairClient hostRepairClient() {
+        return repairClient;
+    }
+
     public ConfigServerMock configServerMock() {
         return configServerMock;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java
new file mode 100644
index 00000000000..556755581fe
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java
@@ -0,0 +1,51 @@
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author olaa
+ */
+public class HostRepairMaintainerTest {
+
+    private final ControllerTester tester = new ControllerTester();
+    private final HostRepairMaintainer maintainer = new HostRepairMaintainer(tester.controller(), Duration.ofHours(12));
+
+    @Test
+    public void maintain() {
+        var zoneId = ZoneId.from("dev.us-east-1");
+        var hostname1 = HostName.from("node-1-tenant-host-dev.us-east-1");
+        var hostname2 = HostName.from("node-2-tenant-host-dev.us-east-1");
+        var timestamp = Instant.now().toEpochMilli();
+        var openTicket = new RepairTicketReport("OPEN", "ticket-1", timestamp, timestamp);
+        var closedTicket = new RepairTicketReport("CLOSED", "ticket-2", timestamp, timestamp);
+
+        tester.configServer().nodeRepository().addReport(
+                zoneId,
+                hostname1,
+                RepairTicketReport.getReportId(),
+                openTicket.toJsonNode());
+        tester.configServer().nodeRepository().addReport(
+                zoneId,
+                hostname2,
+                RepairTicketReport.getReportId(),
+                closedTicket.toJsonNode());
+
+        maintainer.maintain();
+        var updatedNodes = tester.serviceRegistry().hostRepairClient().getUpdatedNodes();
+        assertEquals(1, updatedNodes.size());
+        assertEquals(hostname1, updatedNodes.get(0).hostname());
+    }
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 385f0fbc3cf..bb3578b2482 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -28,6 +28,9 @@
       "name": "DeploymentMetricsMaintainer"
     },
     {
+      "name": "HostRepairMaintainer"
+    },
+    {
       "name": "JobRunner"
     },
     {