diff options
author | toby <smorgrav@yahoo-inc.com> | 2017-09-28 14:17:09 +0200 |
---|---|---|
committer | toby <smorgrav@yahoo-inc.com> | 2017-10-10 13:39:35 +0200 |
commit | 44b4159802098abb76ea6dafd54bfac99458c0d7 (patch) | |
tree | 56b3cb44798a021cf618830aca64f82652b4c41a /controller-server | |
parent | 5a187753f9913514d794852e471ddeffe1fdeb18 (diff) |
Add mainainers to annotate deployments with cluster info and utilization
Diffstat (limited to 'controller-server')
7 files changed, 343 insertions, 8 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 2ce2e480fc5..79994b6ad9c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -332,7 +332,16 @@ public class ApplicationController { configserverClient.prepare(deploymentId, options, rotationInDns.cnames(), rotationInDns.rotations(), applicationPackage.zippedContent()); preparedApplication.activate(); - application = application.with(new Deployment(zone, revision, version, clock.instant())); + + Deployment previousDeployment = application.deployments().get(zone); + Deployment newDeployment = previousDeployment; + if (previousDeployment == null) { + newDeployment = new Deployment(zone, revision, version, clock.instant(), new HashMap<>(), new HashMap<>()); + } else { + newDeployment = new Deployment(zone, revision, version, clock.instant(), previousDeployment.clusterUtils(), previousDeployment.clusterInfo()); + } + + application = application.with(newDeployment); store(application, lock); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java index 569e97e4f53..60f8bcfe3d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java @@ -16,7 +16,7 @@ public class ClusterInfo { private final ClusterSpec.Type clusterType; private final List<String> hostnames; - ClusterInfo(Flavor flavor, ClusterSpec.Type clusterType, List<String> hostnames) { + public ClusterInfo(Flavor flavor, ClusterSpec.Type clusterType, List<String> hostnames) { this.flavor = flavor; this.clusterType = clusterType; this.hostnames = hostnames; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index f537206d0fb..915f3e81b90 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -21,10 +21,14 @@ public class Deployment { private final ApplicationRevision revision; private final Version version; private final Instant deployTime; - private final Map<Id, ClusterUtilization> clusterUtils = new HashMap<>(); - private final Map<Id, ClusterInfo> clusterInfo = new HashMap<>(); + private final Map<Id, ClusterUtilization> clusterUtils; + private final Map<Id, ClusterInfo> clusterInfo; public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime) { + this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>()); + } + + public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime, Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo) { Objects.requireNonNull(zone, "zone cannot be null"); Objects.requireNonNull(revision, "revision cannot be null"); Objects.requireNonNull(version, "version cannot be null"); @@ -33,6 +37,8 @@ public class Deployment { this.revision = revision; this.version = version; this.deployTime = deployTime; + this.clusterUtils = clusterUtils; + this.clusterInfo = clusterInfo; } /** Returns the zone this was deployed to */ @@ -47,12 +53,20 @@ public class Deployment { /** Returns the time this was deployed */ public Instant at() { return deployTime; } - public Map<Id, ClusterUtilization> getClusterUtilization() { + public Map<Id, ClusterInfo> clusterInfo() { + return clusterInfo; + } + + public Map<Id, ClusterUtilization> clusterUtils() { return clusterUtils; } - public Map<Id, ClusterInfo> getClusterInfo() { - return clusterInfo; + public Deployment withClusterUtils(Map<Id, ClusterUtilization> clusterUtilization) { + return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo); + } + + public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) { + return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java new file mode 100644 index 00000000000..af512db67f5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -0,0 +1,251 @@ +package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.ClusterInfo; +import com.yahoo.vespa.hosted.controller.application.Deployment; + +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Fetch info about hardware, hostnames and cluster specifications and update applications. + * + * @author smorgrav + */ +public class ClusterInfoMaintainer extends Maintainer { + + private final Controller controller; + + public ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl) { + super(controller, duration, jobControl); + this.controller = controller; + } + + private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeRepositoryJsonModel nodes, NodeFlavors flavors) { + Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>(); + + Map<String, List<NodeRepositoryJsonModel.Node>> clusters = nodes.nodes.stream() + .filter(node -> node.membership != null) + .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterid)); + + for (String id : clusters.keySet()) { + List<NodeRepositoryJsonModel.Node> clusterNodes = clusters.get(id); + + //Assume they are all equal and use first node as a representatitve for the cluster + NodeRepositoryJsonModel.Node node = clusterNodes.get(0); + + Optional<Flavor> flavorOpt = flavors.getFlavor(node.nodeFlavor); + + List<String> hostnames = clusterNodes.stream().map(node1 -> { return node1.hostname; }).collect(Collectors.toList()); + + ClusterInfo inf = new ClusterInfo(flavorOpt.get(), ClusterSpec.Type.from(node.membership.clusterType), hostnames); + + infoMap.put(new ClusterSpec.Id(id), inf); + } + + return infoMap; + } + + private static String clusterid(NodeRepositoryJsonModel.Node node) { + return node.membership.clusterId; + + } + + private NodeRepositoryJsonModel getApplicationNodes(ApplicationId appId, Zone zone) { + NodeRepositoryJsonModel nodesResponse = null; + ObjectMapper mapper = new ObjectMapper(); + for (URI uri : controller.getConfigServerUris(null, null)) { + try { + nodesResponse = mapper.readValue(uri.toURL(), NodeRepositoryJsonModel.class); + break; + } catch (IOException ioe) { + + } + } + return nodesResponse; + } + + @Override + protected void maintain() { + for (Application application : controller().applications().asList()) { + for (Deployment deployment : application.deployments().values()) { + Zone zone = deployment.zone(); + if (zone.nodeFlavors().isPresent()) { + NodeFlavors flavors = zone.nodeFlavors().get(); + NodeRepositoryJsonModel appNodes = getApplicationNodes(application.id(), zone); + + Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(appNodes, flavors); + application.with(deployment.withClusterInfo(clusterInfo)); + } + } + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class NodeRepositoryJsonModel { + + public final List<Node> nodes; + + @JsonCreator + public NodeRepositoryJsonModel(@JsonProperty("nodes") List<Node> nodes) { + this.nodes = Collections.unmodifiableList(nodes); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Node { + + public final String hostname; + public final String wantedDockerImage; + public final String currentDockerImage; + public final String nodeState; + public final String nodeType; + public final String nodeFlavor; + public final String nodeCanonicalFlavor; + public final String wantedVespaVersion; + public final String vespaVersion; + public final Owner owner; + public final Membership membership; + public final Long wantedRestartGeneration; + public final Long currentRestartGeneration; + public final Long wantedRebootGeneration; + public final Long currentRebootGeneration; + public final Double minCpuCores; + public final Double minMainMemoryAvailableGb; + public final Double minDiskAvailableGb; + + @JsonCreator + public Node(@JsonProperty("id") String hostname, + @JsonProperty("wantedDockerImage") String wantedDockerImage, + @JsonProperty("currentDockerImage") String currentDockerImage, + @JsonProperty("state") String nodeState, + @JsonProperty("type") String nodeType, + @JsonProperty("flavor") String nodeFlavor, + @JsonProperty("canonicalFlavor") String nodeCanonicalFlavor, + @JsonProperty("wantedVespaVersion") String wantedVespaVersion, + @JsonProperty("vespaVersion") String vespaVersion, + @JsonProperty("owner") Owner owner, + @JsonProperty("membership") Membership membership, + @JsonProperty("restartGeneration") Long wantedRestartGeneration, + @JsonProperty("currentRestartGeneration") Long currentRestartGeneration, + @JsonProperty("rebootGeneration") Long wantedRebootGeneration, + @JsonProperty("currentRebootGeneration") Long currentRebootGeneration, + @JsonProperty("minCpuCores") Double minCpuCores, + @JsonProperty("minMainMemoryAvailableGb") Double minMainMemoryAvailableGb, + @JsonProperty("minDiskAvailableGb") Double minDiskAvailableGb) { + this.hostname = hostname; + this.wantedDockerImage = wantedDockerImage; + this.currentDockerImage = currentDockerImage; + this.nodeState = nodeState; + this.nodeType = nodeType; + this.nodeFlavor = nodeFlavor; + this.nodeCanonicalFlavor = nodeCanonicalFlavor; + this.wantedVespaVersion = wantedVespaVersion; + this.vespaVersion = vespaVersion; + this.owner = owner; + this.membership = membership; + this.wantedRestartGeneration = wantedRestartGeneration; + this.currentRestartGeneration = currentRestartGeneration; + this.wantedRebootGeneration = wantedRebootGeneration; + this.currentRebootGeneration = currentRebootGeneration; + this.minCpuCores = minCpuCores; + this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; + this.minDiskAvailableGb = minDiskAvailableGb; + } + + public String toString() { + return "Node {" + + " containerHostname = " + hostname + + " wantedDockerImage = " + wantedDockerImage + + " currentDockerImage = " + currentDockerImage + + " nodeState = " + nodeState + + " nodeType = " + nodeType + + " nodeFlavor = " + nodeFlavor + + " wantedVespaVersion = " + wantedVespaVersion + + " vespaVersion = " + vespaVersion + + " owner = " + owner + + " membership = " + membership + + " wantedRestartGeneration = " + wantedRestartGeneration + + " currentRestartGeneration = " + currentRestartGeneration + + " wantedRebootGeneration = " + wantedRebootGeneration + + " currentRebootGeneration = " + currentRebootGeneration + + " minCpuCores = " + minCpuCores + + " minMainMemoryAvailableGb = " + minMainMemoryAvailableGb + + " minDiskAvailableGb = " + minDiskAvailableGb + + " }"; + } + + + public static class Owner { + public final String tenant; + public final String application; + public final String instance; + + public Owner( + @JsonProperty("tenant") String tenant, + @JsonProperty("application") String application, + @JsonProperty("instance") String instance) { + this.tenant = tenant; + this.application = application; + this.instance = instance; + } + + public String toString() { + return "Owner {" + + " tenant = " + tenant + + " application = " + application + + " instance = " + instance + + " }"; + } + } + + public static class Membership { + public final String clusterType; + public final String clusterId; + public final String group; + public final int index; + public final boolean retired; + + public Membership( + @JsonProperty("clustertype") String clusterType, + @JsonProperty("clusterid") String clusterId, + @JsonProperty("group") String group, + @JsonProperty("index") int index, + @JsonProperty("retired") boolean retired) { + this.clusterType = clusterType; + this.clusterId = clusterId; + this.group = group; + this.index = index; + this.retired = retired; + } + + @Override + public String toString() { + return "Membership {" + + " clusterType = " + clusterType + + " clusterId = " + clusterId + + " group = " + group + + " index = " + index + + " retired = " + retired + + " }"; + } + } + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java new file mode 100644 index 00000000000..0986b0abf7c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -0,0 +1,53 @@ +package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; +import com.yahoo.vespa.hosted.controller.application.Deployment; + +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Fetch utilization metrics and update applications with this data. + * + * @author smorgrav + */ +public class ClusterUtilizationMaintainer extends Maintainer { + + Controller controller; + + public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) { + super(controller, duration, jobControl); + this.controller = controller; + } + + private Map<ClusterSpec.Id, ClusterUtilization> getUpdatedClusterUtilizations(ApplicationId app, Zone zone) { + List<MetricsService.ClusterCostMetrics> systemMetrics = controller.metricsService().getClusterCostMetrics(app, zone); + + Map<ClusterSpec.Id, ClusterUtilization> utilizationMap = new HashMap<>(); + for (MetricsService.ClusterCostMetrics clusterCostMetrics : systemMetrics) { + MetricsService.CostMetrics systemMetric = clusterCostMetrics.costMetrics(); + ClusterUtilization utilization = new ClusterUtilization(systemMetric.memUtil() / 100, systemMetric.cpuUtil() / 100, systemMetric.diskUtil(), 0); + utilizationMap.put(new ClusterSpec.Id(clusterCostMetrics.clusterId()), utilization); + } + + return utilizationMap; + } + + @Override + protected void maintain() { + for (Application application : controller().applications().asList()) { + for (Deployment deployment : application.deployments().values()) { + Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone()); + application.with(deployment.withClusterUtils(clusterUtilization)); + } + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 7788821f04d..880abaaa6f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -33,6 +33,8 @@ public class ControllerMaintenance extends AbstractComponent { private final Upgrader upgrader; private final DelayedDeployer delayedDeployer; private final BlockedChangeDeployer blockedChangeDeployer; + private final ClusterInfoMaintainer clusterInfoMaintainer; + private final ClusterUtilizationMaintainer clusterUtilizationMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, @@ -50,6 +52,8 @@ public class ControllerMaintenance extends AbstractComponent { upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); delayedDeployer = new DelayedDeployer(controller, maintenanceInterval, jobControl); blockedChangeDeployer = new BlockedChangeDeployer(controller, maintenanceInterval, jobControl); + clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl); + clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); } public Upgrader upgrader() { return upgrader; } @@ -68,6 +72,8 @@ public class ControllerMaintenance extends AbstractComponent { upgrader.deconstruct(); delayedDeployer.deconstruct(); blockedChangeDeployer.deconstruct(); + clusterUtilizationMaintainer.deconstruct(); + clusterInfoMaintainer.deconstruct(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 81b3cb635ef..76e0d3bf6db 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.application.SourceRevision; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Optional; @@ -191,7 +192,8 @@ public class ApplicationSerializer { return new Deployment(zoneFromSlime(deploymentObject.field(zoneField)), applicationRevisionFromSlime(deploymentObject.field(applicationPackageRevisionField)).get(), Version.fromString(deploymentObject.field(versionField).asString()), - Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong())); + Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), + new HashMap<>(), new HashMap<>()); //TODO } private Zone zoneFromSlime(Inspector object) { |