Create maintainer tracking host repair status

Extended ZmsClient to add/delete policy rules
Extended controller's node repo client with patchNode function

18 files changed, 421 insertions, 3 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
index a522e26a46d..7b4d82a9f53 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandl
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
@@ -79,4 +80,5 @@ public interface ServiceRegistry {
 
     BillingController billingController();
 
+    HostRepairClient hostRepairClient();
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java
index ec5d62569f6..942f0f35f58 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java
@@ -136,6 +136,17 @@ public class ZmsClientMock implements ZmsClient {
     }
 
     @Override
+    public void addPolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole) {
+
+    }
+
+    @Override
+    public boolean deletePolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole) {
+        return false;
+    }
+
+
+    @Override
     public void close() {}
 
     private static AthenzDomain getTenantDomain(AthenzResourceName resource) {
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
index 07e411cd5cd..b57b2dbc496 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.DockerImage;
@@ -10,6 +11,8 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.TenantName;
 
 import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -48,13 +51,14 @@ public class Node {
     private final boolean wantToRetire;
     private final boolean wantToDeprovision;
     private final Optional<TenantName> reservedTo;
+    private final Map<String, JsonNode> reports;
 
     public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner,
                 Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion,
                 Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState,
                 Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration,
                 int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision,
-                Optional<TenantName> reservedTo, DockerImage wantedDockerImage, DockerImage currentDockerImage) {
+                Optional<TenantName> reservedTo, DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports) {
         this.hostname = hostname;
         this.parentHostname = parentHostname;
         this.state = state;
@@ -82,6 +86,7 @@ public class Node {
         this.reservedTo = reservedTo;
         this.wantedDockerImage = wantedDockerImage;
         this.currentDockerImage = currentDockerImage;
+        this.reports = reports;
     }
 
     public HostName hostname() {
@@ -188,6 +193,10 @@ public class Node {
 
     public Optional<TenantName> reservedTo() { return reservedTo; }
 
+    public Map<String, JsonNode> reports() {
+        return reports;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -258,6 +267,7 @@ public class Node {
         private boolean wantToRetire;
         private boolean wantToDeprovision;
         private Optional<TenantName> reservedTo = Optional.empty();
+        private Map<String, JsonNode> reports = new HashMap<>();
 
         public Builder() { }
 
@@ -289,6 +299,7 @@ public class Node {
             this.wantToRetire = node.wantToRetire;
             this.wantToDeprovision = node.wantToDeprovision;
             this.reservedTo = node.reservedTo;
+            this.reports = node.reports;
         }
 
         public Builder hostname(HostName hostname) {
@@ -431,7 +442,7 @@ public class Node {
                             currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState,
                             suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration,
                             cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo,
-                            wantedDockerImage, currentDockerImage);
+                            wantedDockerImage, currentDockerImage, reports);
         }
 
     }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
index aebfab7cbff..6f4b39ac9b9 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
@@ -90,6 +90,8 @@ public interface NodeRepository {
 
     void retireAndDeprovision(ZoneId zoneId, String hostName);
 
+    void patchNode(ZoneId zoneId, String hostName, NodeRepositoryNode node);
+
     private static Node toNode(NodeRepositoryNode node) {
         var application = Optional.ofNullable(node.getOwner())
                                   .map(owner -> ApplicationId.from(owner.getTenant(), owner.getApplication(),
@@ -128,7 +130,8 @@ public interface NodeRepository {
                         node.getWantToDeprovision(),
                         Optional.ofNullable(node.getReservedTo()).map(TenantName::from),
                         dockerImageFrom(node.getWantedDockerImage()),
-                        dockerImageFrom(node.getCurrentDockerImage()));
+                        dockerImageFrom(node.getCurrentDockerImage()),
+                        node.getReports());
     }
 
     private static String clusterIdOf(NodeMembership nodeMembership) {
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java
new file mode 100644
index 00000000000..e79928f59d4
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java
@@ -0,0 +1,19 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.repair;
+
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author olaa
+ */
+public interface HostRepairClient {
+
+    /* Checks current ticket status and takes appropriate action */
+    void updateRepairStatus(ZoneApi zone, Map<Node, RepairTicketReport> nodes);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java
new file mode 100644
index 00000000000..307ba7408c5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java
@@ -0,0 +1,26 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.repair;
+
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author olaa
+ */
+public class MockRepairClient implements HostRepairClient {
+
+    List<Node> updatedNodes = new ArrayList<>();
+
+    @Override
+    public void updateRepairStatus(ZoneApi zone, Map<Node, RepairTicketReport> nodes) {
+        updatedNodes.addAll(nodes.keySet());
+    }
+
+    public List<Node> getUpdatedNodes() {
+        return updatedNodes;
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java
new file mode 100644
index 00000000000..c44e00706e2
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java
@@ -0,0 +1,56 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.repair;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author olaa
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RepairTicketReport {
+
+    private static final String REPORT_ID = "repair-ticket";
+    private static final ObjectMapper objectMapper = new ObjectMapper();@JsonIgnore
+
+    public String status;
+    public String ticketNumber;
+    public long createdMillis;
+
+    public RepairTicketReport(@JsonProperty("status") String status,
+                              @JsonProperty("ticketNumber") String ticketNumber,
+                              @JsonProperty("createdMillis") long createdMillis) {
+        this.status = status;
+        this.ticketNumber = ticketNumber;
+        this.createdMillis = createdMillis;
+    }
+
+    public String getStatus() {
+        return status;
+    }
+
+    public String getTicketNumber() {
+        return ticketNumber;
+    }
+
+    public long getCreatedMillis() {
+        return createdMillis;
+    }
+
+    public static String getReportId() {
+        return REPORT_ID;
+    }
+
+    public static RepairTicketReport fromJsonNode(JsonNode node) {
+        return uncheck(() -> objectMapper.treeToValue(node, RepairTicketReport.class));
+    }
+
+    public JsonNode toJsonNode() {
+        return uncheck(() -> objectMapper.valueToTree(this));
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/package-info.java
new file mode 100644
index 00000000000..f53cb1ee43c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/package-info.java
@@ -0,0 +1,5 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.repair;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 0e72a1b42a7..6731c30ecd7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -45,6 +45,8 @@ public class ControllerMaintenance extends AbstractComponent {
     private final ResourceTagMaintainer resourceTagMaintainer;
     private final SystemRoutingPolicyMaintainer systemRoutingPolicyMaintainer;
     private final ApplicationMetaDataGarbageCollector applicationMetaDataGarbageCollector;
+    private final HostRepairMaintainer hostRepairMaintainer;
+
 
     @Inject
     @SuppressWarnings("unused") // instantiated by Dependency Injection
@@ -75,6 +77,7 @@ public class ControllerMaintenance extends AbstractComponent {
         resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger());
         systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10));
         applicationMetaDataGarbageCollector = new ApplicationMetaDataGarbageCollector(controller, Duration.ofHours(12));
+        hostRepairMaintainer = new HostRepairMaintainer(controller, Duration.ofHours(12));
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -102,6 +105,7 @@ public class ControllerMaintenance extends AbstractComponent {
         rotationStatusUpdater.close();
         resourceTagMaintainer.close();
         systemRoutingPolicyMaintainer.close();
+        hostRepairMaintainer.close();
     }
 
     /** Create one OS upgrader per cloud found in the zone registry of controller */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java
new file mode 100644
index 00000000000..c3c5f902dff
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java
@@ -0,0 +1,81 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ *
+ * Responsible for keeping track of hosts under repair.
+ *
+ * @author olaa
+ */
+public class HostRepairMaintainer extends ControllerMaintainer {
+
+    private final NodeRepository nodeRepository;
+    private final HostRepairClient repairClient;
+
+    private static final Logger log = Logger.getLogger(HostRepairMaintainer.class.getName());
+
+
+    public HostRepairMaintainer(Controller controller, Duration interval) {
+        super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)));
+        this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
+        this.repairClient = controller.serviceRegistry().hostRepairClient();
+    }
+
+
+    @Override
+    protected boolean maintain() {
+        AtomicInteger exceptions = new AtomicInteger(0);
+
+        controller().zoneRegistry().zones()
+                .reachable().zones().stream()
+                .forEach(zoneApi -> {
+                            var nodeTicketMap = nodeRepository.list((zoneApi).getId())
+                                    .stream()
+                                    .filter(this::hasOpenTicket)
+                                    .collect(Collectors.toMap(
+                                            node -> node,
+                                            this::getTicketReport)
+                                    );
+                            try {
+                                repairClient.updateRepairStatus(zoneApi, nodeTicketMap);
+                            } catch (Exception e) {
+                                log.warning("Failed to update repair status; " + Exceptions.toMessageString(e));
+                                exceptions.incrementAndGet();
+                            }
+                        }
+                );
+
+        return exceptions.get() == 0;
+    }
+
+
+    private boolean hasOpenTicket(Node node) {
+        var reports = node.reports();
+        if (!reports.containsKey(RepairTicketReport.getReportId())) {
+            return false;
+        }
+        return getTicketReport(node).getStatus().equals("OPEN");
+    }
+
+    private RepairTicketReport getTicketReport(Node node) {
+        return uncheck(() -> RepairTicketReport.fromJsonNode(node.reports().get(RepairTicketReport.getReportId())));
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
index 90276b6b590..72cc000ef98 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
@@ -226,6 +227,11 @@ public class NodeRepositoryMock implements NodeRepository {
         nodeRepository.get(zoneId).remove(HostName.from(hostName));
     }
 
+    @Override
+    public void patchNode(ZoneId zoneId, String hostName, NodeRepositoryNode node) {
+        throw new UnsupportedOperationException();
+    }
+
     public Optional<Duration> osUpgradeBudget(ZoneId zone, NodeType type, Version version) {
         return Optional.ofNullable(osUpgradeBudgets.get(Objects.hash(zone, type, version)));
     }
@@ -264,4 +270,8 @@ public class NodeRepositoryMock implements NodeRepository {
         modifyNodes(deployment, hostname, node -> new Node.Builder(node).rebootGeneration(node.rebootGeneration() + 1).build());
     }
 
+    public void addReport(ZoneId zoneId, HostName hostName, String reportId, JsonNode report) {
+        nodeRepository.get(zoneId).get(hostName).reports().put(reportId, report);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index 1b21f7db7c4..3ec02c6ceb7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -20,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
 import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService;
@@ -61,6 +63,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final MockResourceTagger mockResourceTagger = new MockResourceTagger();
     private final ApplicationRoleService applicationRoleService = new NoopApplicationRoleService();
     private final BillingController billingController = new MockBillingController();
+    private final MockRepairClient repairClient = new MockRepairClient();
 
     public ServiceRegistryMock(SystemName system) {
         this.zoneRegistryMock = new ZoneRegistryMock(system);
@@ -192,6 +195,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
         return billingController;
     }
 
+    @Override
+    public MockRepairClient hostRepairClient() {
+        return repairClient;
+    }
+
     public ConfigServerMock configServerMock() {
         return configServerMock;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java
new file mode 100644
index 00000000000..24094892640
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java
@@ -0,0 +1,49 @@
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient;
+import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author olaa
+ */
+public class HostRepairMaintainerTest {
+
+    private final ControllerTester tester = new ControllerTester();
+    private final HostRepairMaintainer maintainer = new HostRepairMaintainer(tester.controller(), Duration.ofHours(12));
+
+    @Test
+    public void maintain() {
+        var zoneId = ZoneId.from("dev.us-east-1");
+        var hostname1 = HostName.from("node-1-tenant-host-dev.us-east-1");
+        var hostname2 = HostName.from("node-2-tenant-host-dev.us-east-1");
+        var openTicket = new RepairTicketReport("OPEN", "ticket-1", 123l);
+        var closedTicket = new RepairTicketReport("CLOSED", "ticket-2", 123l);
+
+        tester.configServer().nodeRepository().addReport(
+                zoneId,
+                hostname1,
+                RepairTicketReport.getReportId(),
+                openTicket.toJsonNode());
+        tester.configServer().nodeRepository().addReport(
+                zoneId,
+                hostname2,
+                RepairTicketReport.getReportId(),
+                closedTicket.toJsonNode());
+
+        maintainer.maintain();
+        var updatedNodes = tester.serviceRegistry().hostRepairClient().getUpdatedNodes();
+        assertEquals(1, updatedNodes.size());
+        assertEquals(hostname1, updatedNodes.get(0).hostname());
+    }
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 385f0fbc3cf..bb3578b2482 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -28,6 +28,9 @@
       "name": "DeploymentMetricsMaintainer"
     },
     {
+      "name": "HostRepairMaintainer"
+    },
+    {
       "name": "JobRunner"
     },
     {
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
index eaf83238145..9a31ea81715 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java
@@ -9,8 +9,10 @@ import com.yahoo.vespa.athenz.api.OktaAccessToken;
 import com.yahoo.vespa.athenz.api.OktaIdentityToken;
 import com.yahoo.vespa.athenz.client.common.ClientBase;
 import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity;
+import com.yahoo.vespa.athenz.client.zms.bindings.AssertionEntity;
 import com.yahoo.vespa.athenz.client.zms.bindings.DomainListResponseEntity;
 import com.yahoo.vespa.athenz.client.zms.bindings.MembershipResponseEntity;
+import com.yahoo.vespa.athenz.client.zms.bindings.PolicyEntity;
 import com.yahoo.vespa.athenz.client.zms.bindings.ProviderResourceGroupRolesRequestEntity;
 import com.yahoo.vespa.athenz.client.zms.bindings.TenancyRequestEntity;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
@@ -23,6 +25,8 @@ import javax.net.ssl.SSLContext;
 import java.net.URI;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
+import java.util.OptionalInt;
 import java.util.Set;
 import java.util.function.Supplier;
 
@@ -149,6 +153,47 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient {
         });
     }
 
+    @Override
+    public void addPolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole) {
+        URI uri = zmsUrl.resolve(String.format("domain/%s/policy/%s/assertion",
+                athenzDomain.getName(), athenzPolicy));
+        HttpUriRequest request = RequestBuilder.put()
+                .setUri(uri)
+                .setEntity(toJsonStringEntity(new AssertionEntity(athenzRole.toResourceNameString(), resourceName.toResourceNameString(), action)))
+                .build();
+        execute(request, response -> readEntity(response, Void.class));
+    }
+
+    @Override
+    public boolean deletePolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole) {
+        URI uri = zmsUrl.resolve(String.format("domain/%s/policy/%s",
+                athenzDomain.getName(), athenzPolicy));
+        HttpUriRequest request = RequestBuilder.get()
+                .setUri(uri)
+                .build();
+        PolicyEntity policyEntity =  execute(request, response -> readEntity(response, PolicyEntity.class));
+
+        OptionalInt assertionId = policyEntity.getAssertions().stream()
+                .filter(assertionEntity -> assertionEntity.getAction().equals(action) &&
+                        assertionEntity.getResource().equals(resourceName.toResourceNameString()) &&
+                        assertionEntity.getRole().equals(athenzRole.toResourceNameString()))
+                .mapToInt(AssertionEntity::getAssertionId).findFirst();
+
+        if (assertionId.isEmpty()) {
+            return false;
+        }
+
+        uri = zmsUrl.resolve(String.format("domain/%s/policy/%s/assertion/d",
+                athenzDomain.getName(), athenzPolicy, assertionId.getAsInt()));
+
+        request = RequestBuilder.delete()
+                .setUri(uri)
+                .build();
+
+        execute(request, response -> readEntity(response, Void.class));
+        return true;
+    }
+
     private static Header createCookieHeaderWithOktaTokens(OktaIdentityToken identityToken, OktaAccessToken accessToken) {
         return new BasicHeader("Cookie", String.format("okta_at=%s; okta_it=%s", accessToken.token(), identityToken.token()));
     }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java
index 12762534bd4..c7f865a58bb 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java
@@ -38,5 +38,9 @@ public interface ZmsClient extends AutoCloseable {
 
     boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity);
 
+    void addPolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole);
+
+    boolean deletePolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole);
+
     void close();
 }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AssertionEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AssertionEntity.java
new file mode 100644
index 00000000000..b181bc7a8c7
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AssertionEntity.java
@@ -0,0 +1,50 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.client.zms.bindings;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author olaa
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class AssertionEntity {
+
+    private final String role;
+    private final String resource;
+    private final String action;
+    private final Integer assertionId;
+
+
+    public AssertionEntity(String role, String resource, String action) {
+        this(role, resource, action, null);
+    }
+
+    public AssertionEntity(@JsonProperty("role") String role,
+                           @JsonProperty("resource") String resource,
+                           @JsonProperty("action") String action,
+                           @JsonProperty("assertionId") Integer assertionId) {
+        this.role = role;
+        this.resource = resource;
+        this.action = action;
+        this.assertionId = assertionId;
+    }
+
+    public String getRole() {
+        return role;
+    }
+
+    public String getResource() {
+        return resource;
+    }
+
+    public String getAction() {
+        return action;
+    }
+
+    @JsonIgnore
+    public int getAssertionId() {
+        return assertionId;
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java
new file mode 100644
index 00000000000..442f9d12c71
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java
@@ -0,0 +1,31 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.client.zms.bindings;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * @author olaa
+ */
+public class PolicyEntity {
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    private final List<AssertionEntity> assertions;
+    private final String name;
+
+    public PolicyEntity(@JsonProperty("name") String name,
+                        @JsonProperty("assertions") List<AssertionEntity> assertions) {
+        this.name = name;
+        this.assertions = assertions;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public List<AssertionEntity> getAssertions() {
+        return assertions;
+    }
+}