diff options
21 files changed, 448 insertions, 11 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index a522e26a46d..7b4d82a9f53 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandl import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; +import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; @@ -79,4 +80,5 @@ public interface ServiceRegistry { BillingController billingController(); + HostRepairClient hostRepairClient(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index ec5d62569f6..942f0f35f58 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -136,6 +136,17 @@ public class ZmsClientMock implements ZmsClient { } @Override + public void addPolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole) { + + } + + @Override + public boolean deletePolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole) { + return false; + } + + + @Override public void close() {} private static AthenzDomain getTenantDomain(AthenzResourceName resource) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index 07e411cd5cd..b57b2dbc496 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; +import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; @@ -10,6 +11,8 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -48,13 +51,14 @@ public class Node { private final boolean wantToRetire; private final boolean wantToDeprovision; private final Optional<TenantName> reservedTo; + private final Map<String, JsonNode> reports; public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision, - Optional<TenantName> reservedTo, DockerImage wantedDockerImage, DockerImage currentDockerImage) { + Optional<TenantName> reservedTo, DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports) { this.hostname = hostname; this.parentHostname = parentHostname; this.state = state; @@ -82,6 +86,7 @@ public class Node { this.reservedTo = reservedTo; this.wantedDockerImage = wantedDockerImage; this.currentDockerImage = currentDockerImage; + this.reports = reports; } public HostName hostname() { @@ -188,6 +193,10 @@ public class Node { public Optional<TenantName> reservedTo() { return reservedTo; } + public Map<String, JsonNode> reports() { + return reports; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -258,6 +267,7 @@ public class Node { private boolean wantToRetire; private boolean wantToDeprovision; private Optional<TenantName> reservedTo = Optional.empty(); + private Map<String, JsonNode> reports = new HashMap<>(); public Builder() { } @@ -289,6 +299,7 @@ public class Node { this.wantToRetire = node.wantToRetire; this.wantToDeprovision = node.wantToDeprovision; this.reservedTo = node.reservedTo; + this.reports = node.reports; } public Builder hostname(HostName hostname) { @@ -431,7 +442,7 @@ public class Node { currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo, - wantedDockerImage, currentDockerImage); + wantedDockerImage, currentDockerImage, reports); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index aebfab7cbff..6f4b39ac9b9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -90,6 +90,8 @@ public interface NodeRepository { void retireAndDeprovision(ZoneId zoneId, String hostName); + void patchNode(ZoneId zoneId, String hostName, NodeRepositoryNode node); + private static Node toNode(NodeRepositoryNode node) { var application = Optional.ofNullable(node.getOwner()) .map(owner -> ApplicationId.from(owner.getTenant(), owner.getApplication(), @@ -128,7 +130,8 @@ public interface NodeRepository { node.getWantToDeprovision(), Optional.ofNullable(node.getReservedTo()).map(TenantName::from), dockerImageFrom(node.getWantedDockerImage()), - dockerImageFrom(node.getCurrentDockerImage())); + dockerImageFrom(node.getCurrentDockerImage()), + node.getReports()); } private static String clusterIdOf(NodeMembership nodeMembership) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java new file mode 100644 index 00000000000..e79928f59d4 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.repair; + +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; + +import java.util.List; +import java.util.Map; + +/** + * @author olaa + */ +public interface HostRepairClient { + + /* Checks current ticket status and takes appropriate action */ + void updateRepairStatus(ZoneApi zone, Map<Node, RepairTicketReport> nodes); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java new file mode 100644 index 00000000000..307ba7408c5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.repair; + +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author olaa + */ +public class MockRepairClient implements HostRepairClient { + + List<Node> updatedNodes = new ArrayList<>(); + + @Override + public void updateRepairStatus(ZoneApi zone, Map<Node, RepairTicketReport> nodes) { + updatedNodes.addAll(nodes.keySet()); + } + + public List<Node> getUpdatedNodes() { + return updatedNodes; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java new file mode 100644 index 00000000000..c2425fe0f72 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java @@ -0,0 +1,63 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.repair; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author olaa + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class RepairTicketReport { + + private static final String REPORT_ID = "repairTicket"; + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public String status; + public String ticketNumber; + public long createdMillis; + public long updatedMillis; + + public RepairTicketReport(@JsonProperty("status") String status, + @JsonProperty("ticketNumber") String ticketNumber, + @JsonProperty("createdMillis") long createdMillis, + @JsonProperty("updatedMillis") long updatedMillis) { + this.status = status; + this.ticketNumber = ticketNumber; + this.createdMillis = createdMillis; + this.updatedMillis = updatedMillis; + } + + public String getStatus() { + return status; + } + + public String getTicketNumber() { + return ticketNumber; + } + + public long getCreatedMillis() { + return createdMillis; + } + + public long getUpdatedMillis() { + return updatedMillis; + } + + public static String getReportId() { + return REPORT_ID; + } + + public static RepairTicketReport fromJsonNode(JsonNode node) { + return uncheck(() -> objectMapper.treeToValue(node, RepairTicketReport.class)); + } + + public JsonNode toJsonNode() { + return uncheck(() -> objectMapper.valueToTree(this)); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/package-info.java new file mode 100644 index 00000000000..f53cb1ee43c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/package-info.java @@ -0,0 +1,5 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.repair; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 0e72a1b42a7..6731c30ecd7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -45,6 +45,8 @@ public class ControllerMaintenance extends AbstractComponent { private final ResourceTagMaintainer resourceTagMaintainer; private final SystemRoutingPolicyMaintainer systemRoutingPolicyMaintainer; private final ApplicationMetaDataGarbageCollector applicationMetaDataGarbageCollector; + private final HostRepairMaintainer hostRepairMaintainer; + @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection @@ -75,6 +77,7 @@ public class ControllerMaintenance extends AbstractComponent { resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger()); systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10)); applicationMetaDataGarbageCollector = new ApplicationMetaDataGarbageCollector(controller, Duration.ofHours(12)); + hostRepairMaintainer = new HostRepairMaintainer(controller, Duration.ofHours(12)); } public Upgrader upgrader() { return upgrader; } @@ -102,6 +105,7 @@ public class ControllerMaintenance extends AbstractComponent { rotationStatusUpdater.close(); resourceTagMaintainer.close(); systemRoutingPolicyMaintainer.close(); + hostRepairMaintainer.close(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java new file mode 100644 index 00000000000..e3c6862384f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java @@ -0,0 +1,81 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; +import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport; +import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * + * Responsible for keeping track of hosts under repair. + * + * @author olaa + */ +public class HostRepairMaintainer extends ControllerMaintainer { + + private final NodeRepository nodeRepository; + private final HostRepairClient repairClient; + + private static final Logger log = Logger.getLogger(HostRepairMaintainer.class.getName()); + + + public HostRepairMaintainer(Controller controller, Duration interval) { + super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); + this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + this.repairClient = controller.serviceRegistry().hostRepairClient(); + } + + + @Override + protected boolean maintain() { + AtomicInteger exceptions = new AtomicInteger(0); + + controller().zoneRegistry().zones() + .reachable().zones().stream() + .forEach(zoneApi -> { + var nodeTicketMap = nodeRepository.list((zoneApi).getId()) + .stream() + .filter(this::hasOpenTicket) + .collect(Collectors.toMap( + node -> node, + this::getTicketReport) + ); + try { + repairClient.updateRepairStatus(zoneApi, nodeTicketMap); + } catch (Exception e) { + log.warning("Failed to update repair status; " + Exceptions.toMessageString(e)); + exceptions.incrementAndGet(); + } + } + ); + + return exceptions.get() == 0; + } + + + private boolean hasOpenTicket(Node node) { + var reports = node.reports(); + if (!reports.containsKey(RepairTicketReport.getReportId())) { + return false; + } + return "OPEN".equals(getTicketReport(node).getStatus()); + } + + private RepairTicketReport getTicketReport(Node node) { + return uncheck(() -> RepairTicketReport.fromJsonNode(node.reports().get(RepairTicketReport.getReportId()))); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 90276b6b590..72cc000ef98 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; +import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -226,6 +227,11 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.get(zoneId).remove(HostName.from(hostName)); } + @Override + public void patchNode(ZoneId zoneId, String hostName, NodeRepositoryNode node) { + throw new UnsupportedOperationException(); + } + public Optional<Duration> osUpgradeBudget(ZoneId zone, NodeType type, Version version) { return Optional.ofNullable(osUpgradeBudgets.get(Objects.hash(zone, type, version))); } @@ -264,4 +270,8 @@ public class NodeRepositoryMock implements NodeRepository { modifyNodes(deployment, hostname, node -> new Node.Builder(node).rebootGeneration(node.rebootGeneration() + 1).build()); } + public void addReport(ZoneId zoneId, HostName hostName, String reportId, JsonNode report) { + nodeRepository.get(zoneId).get(hostName).reports().put(reportId, report); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 1b21f7db7c4..3ec02c6ceb7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -20,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; +import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient; +import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; @@ -61,6 +63,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); private final ApplicationRoleService applicationRoleService = new NoopApplicationRoleService(); private final BillingController billingController = new MockBillingController(); + private final MockRepairClient repairClient = new MockRepairClient(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -192,6 +195,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return billingController; } + @Override + public MockRepairClient hostRepairClient() { + return repairClient; + } + public ConfigServerMock configServerMock() { return configServerMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java new file mode 100644 index 00000000000..556755581fe --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java @@ -0,0 +1,51 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient; +import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient; +import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class HostRepairMaintainerTest { + + private final ControllerTester tester = new ControllerTester(); + private final HostRepairMaintainer maintainer = new HostRepairMaintainer(tester.controller(), Duration.ofHours(12)); + + @Test + public void maintain() { + var zoneId = ZoneId.from("dev.us-east-1"); + var hostname1 = HostName.from("node-1-tenant-host-dev.us-east-1"); + var hostname2 = HostName.from("node-2-tenant-host-dev.us-east-1"); + var timestamp = Instant.now().toEpochMilli(); + var openTicket = new RepairTicketReport("OPEN", "ticket-1", timestamp, timestamp); + var closedTicket = new RepairTicketReport("CLOSED", "ticket-2", timestamp, timestamp); + + tester.configServer().nodeRepository().addReport( + zoneId, + hostname1, + RepairTicketReport.getReportId(), + openTicket.toJsonNode()); + tester.configServer().nodeRepository().addReport( + zoneId, + hostname2, + RepairTicketReport.getReportId(), + closedTicket.toJsonNode()); + + maintainer.maintain(); + var updatedNodes = tester.serviceRegistry().hostRepairClient().getUpdatedNodes(); + assertEquals(1, updatedNodes.size()); + assertEquals(hostname1, updatedNodes.get(0).hostname()); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 385f0fbc3cf..bb3578b2482 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -28,6 +28,9 @@ "name": "DeploymentMetricsMaintainer" }, { + "name": "HostRepairMaintainer" + }, + { "name": "JobRunner" }, { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index c73a19bd9e2..eace7457615 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -61,7 +61,7 @@ public class Autoscaler { private Optional<AllocatableClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes, Limits limits, boolean exclusive) { - if (unstable(clusterNodes)) return Optional.empty(); + if (unstable(clusterNodes, nodeRepository)) return Optional.empty(); AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository); @@ -111,10 +111,18 @@ public class Autoscaler { return 20; } - public static boolean unstable(List<Node> nodes) { - return nodes.stream().anyMatch(node -> node.status().wantToRetire() || - node.allocation().get().membership().retired() || - node.allocation().get().isRemovable()); + public static boolean unstable(List<Node> nodes, NodeRepository nodeRepository) { + // The cluster is processing recent changes + if (nodes.stream().anyMatch(node -> node.status().wantToRetire() || + node.allocation().get().membership().retired() || + node.allocation().get().isRemovable())) + return true; + + // A deployment is ongoing + if (nodeRepository.getNodes(nodes.get(0).allocation().get().owner(), Node.State.reserved).size() > 0) + return true; + + return false; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java index b4a63175548..4597fc04e17 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java @@ -56,7 +56,7 @@ public class MetricsV2MetricsFetcher extends AbstractComponent implements Metric NodeList applicationNodes = nodeRepository.list(application).state(Node.State.active); // Do not try to draw conclusions from utilization while unstable - if (Autoscaler.unstable(applicationNodes.asList())) return Collections.emptyList(); + if (Autoscaler.unstable(applicationNodes.asList(), nodeRepository)) return Collections.emptyList(); Optional<Node> metricsV2Container = applicationNodes.container() .matching(node -> expectedUp(node)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 3b01f678982..c0fd7df9b2e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -28,7 +28,6 @@ import java.util.stream.Collectors; */ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { - private final MetricsDb metricsDb; private final Autoscaler autoscaler; private final Deployer deployer; private final Metric metric; @@ -40,7 +39,6 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Duration interval) { super(nodeRepository, interval, metric); this.autoscaler = new Autoscaler(metricsDb, nodeRepository); - this.metricsDb = metricsDb; this.metric = metric; this.deployer = deployer; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index eaf83238145..33cb6d7d5d4 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -9,8 +9,10 @@ import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; import com.yahoo.vespa.athenz.client.common.ClientBase; import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.AssertionEntity; import com.yahoo.vespa.athenz.client.zms.bindings.DomainListResponseEntity; import com.yahoo.vespa.athenz.client.zms.bindings.MembershipResponseEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.PolicyEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ProviderResourceGroupRolesRequestEntity; import com.yahoo.vespa.athenz.client.zms.bindings.TenancyRequestEntity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; @@ -23,6 +25,8 @@ import javax.net.ssl.SSLContext; import java.net.URI; import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.function.Supplier; @@ -149,6 +153,47 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { }); } + @Override + public void addPolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole) { + URI uri = zmsUrl.resolve(String.format("domain/%s/policy/%s/assertion", + athenzDomain.getName(), athenzPolicy)); + HttpUriRequest request = RequestBuilder.put() + .setUri(uri) + .setEntity(toJsonStringEntity(new AssertionEntity(athenzRole.toResourceNameString(), resourceName.toResourceNameString(), action))) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + + @Override + public boolean deletePolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole) { + URI uri = zmsUrl.resolve(String.format("domain/%s/policy/%s", + athenzDomain.getName(), athenzPolicy)); + HttpUriRequest request = RequestBuilder.get() + .setUri(uri) + .build(); + PolicyEntity policyEntity = execute(request, response -> readEntity(response, PolicyEntity.class)); + + OptionalInt assertionId = policyEntity.getAssertions().stream() + .filter(assertionEntity -> assertionEntity.getAction().equals(action) && + assertionEntity.getResource().equals(resourceName.toResourceNameString()) && + assertionEntity.getRole().equals(athenzRole.toResourceNameString())) + .mapToInt(AssertionEntity::getId).findFirst(); + + if (assertionId.isEmpty()) { + return false; + } + + uri = zmsUrl.resolve(String.format("domain/%s/policy/%s/assertion/%d", + athenzDomain.getName(), athenzPolicy, assertionId.getAsInt())); + + request = RequestBuilder.delete() + .setUri(uri) + .build(); + + execute(request, response -> readEntity(response, Void.class)); + return true; + } + private static Header createCookieHeaderWithOktaTokens(OktaIdentityToken identityToken, OktaAccessToken accessToken) { return new BasicHeader("Cookie", String.format("okta_at=%s; okta_it=%s", accessToken.token(), identityToken.token())); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 12762534bd4..c7f865a58bb 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -38,5 +38,9 @@ public interface ZmsClient extends AutoCloseable { boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity); + void addPolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole); + + boolean deletePolicyRule(AthenzDomain athenzDomain, String athenzPolicy, String action, AthenzResourceName resourceName, AthenzRole athenzRole); + void close(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AssertionEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AssertionEntity.java new file mode 100644 index 00000000000..824aa3b4606 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AssertionEntity.java @@ -0,0 +1,52 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author olaa + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class AssertionEntity { + + private final String role; + private final String resource; + private final String action; + private final Integer id; + + + public AssertionEntity(String role, String resource, String action) { + this(role, resource, action, null); + } + + public AssertionEntity(@JsonProperty("role") String role, + @JsonProperty("resource") String resource, + @JsonProperty("action") String action, + @JsonProperty("id") Integer id) { + this.role = role; + this.resource = resource; + this.action = action; + this.id = id; + } + + public String getRole() { + return role; + } + + public String getResource() { + return resource; + } + + public String getAction() { + return action; + } + + @JsonIgnore + public int getId() { + return id; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java new file mode 100644 index 00000000000..ebc0997cb09 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/PolicyEntity.java @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author olaa + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class PolicyEntity { + + @JsonInclude(JsonInclude.Include.NON_EMPTY) + private final List<AssertionEntity> assertions; + private final String name; + + public PolicyEntity(@JsonProperty("name") String name, + @JsonProperty("assertions") List<AssertionEntity> assertions) { + this.name = name; + this.assertions = assertions; + } + + public String getName() { + return name; + } + + public List<AssertionEntity> getAssertions() { + return assertions; + } +} |