summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Polden <mpolden@mpolden.no>2019-08-30 11:26:36 +0200
committerMartin Polden <mpolden@mpolden.no>2019-08-30 11:27:39 +0200
commitc2455394cbbb1f62610e2f0e1d4ceeceed456ac3 (patch)
tree807076d2e7d1f160a4e8541024f9a150b12c4692
parent0847ebeec917d7d0c99e1c037300c793b41ab6d2 (diff)
Use a single node type in controller
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java10
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java60
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java23
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java15
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java37
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java16
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java11
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java40
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java98
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java119
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java64
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java63
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java32
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java17
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java1
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java3
18 files changed, 313 insertions, 306 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
index e6639a33738..c2d39236b8f 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
@@ -36,6 +36,7 @@ public class Node {
private final double diskGb;
private final double bandwidthGbps;
private final boolean fastDisk;
+ private final int cost;
private final String canonicalFlavor;
private final String clusterId;
private final ClusterType clusterType;
@@ -43,7 +44,7 @@ public class Node {
public Node(HostName hostname, State state, NodeType type, Optional<ApplicationId> owner,
Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, ServiceState serviceState,
long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration,
- double vcpu, double memoryGb, double diskGb, double bandwidthGbps, boolean fastDisk, String canonicalFlavor, String clusterId, ClusterType clusterType) {
+ double vcpu, double memoryGb, double diskGb, double bandwidthGbps, boolean fastDisk, int cost, String canonicalFlavor, String clusterId, ClusterType clusterType) {
this.hostname = hostname;
this.state = state;
this.type = type;
@@ -62,6 +63,7 @@ public class Node {
this.diskGb = diskGb;
this.bandwidthGbps = bandwidthGbps;
this.fastDisk = fastDisk;
+ this.cost = cost;
this.canonicalFlavor = canonicalFlavor;
this.clusterId = clusterId;
this.clusterType = clusterType;
@@ -72,7 +74,7 @@ public class Node {
Version currentVersion, Version wantedVersion) {
this(hostname, state, type, owner, currentVersion, wantedVersion,
Version.emptyVersion, Version.emptyVersion, ServiceState.unorchestrated, 0, 0, 0, 0,
- 2, 8, 50, 1, true, "d-2-8-50", "cluster", ClusterType.container);
+ 2, 8, 50, 1, true, 0, "d-2-8-50", "cluster", ClusterType.container);
}
public HostName hostname() {
@@ -145,6 +147,10 @@ public class Node {
return fastDisk;
}
+ public int cost() {
+ return cost;
+ }
+
public String canonicalFlavor() {
return canonicalFlavor;
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
index fd6b5b48d10..19486b6c2c5 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
@@ -34,30 +34,18 @@ public interface NodeRepository {
NodeList listNodes(ZoneId zone, ApplicationId application);
+ /** List all nodes in given zone */
+ default List<Node> list(ZoneId zone) {
+ return listNodes(zone).nodes().stream()
+ .map(NodeRepository::toNode)
+ .collect(Collectors.toUnmodifiableList());
+ }
+
/** List all nodes in zone owned by given application */
default List<Node> list(ZoneId zone, ApplicationId application) {
return listNodes(zone, application).nodes().stream()
- .map(n -> new Node(com.yahoo.config.provision.HostName.from(n.getHostname()),
- fromJacksonState(n.getState()),
- fromJacksonType(n.getType()), Optional.of(application),
- Version.fromString(n.getVespaVersion()),
- Version.fromString(n.getWantedVespaVersion()),
- Version.fromString(n.getCurrentOsVersion()),
- Version.fromString(n.getWantedOsVersion()),
- fromBoolean(n.getAllowedToBeDown()),
- n.getCurrentRestartGeneration(),
- n.getRestartGeneration(),
- n.getCurrentRebootGeneration(),
- n.getRebootGeneration(),
- n.getMinCpuCores(),
- n.getMinMainMemoryAvailableGb(),
- n.getMinDiskAvailableGb(),
- n.getBandwidth() / 1000,
- n.getFastDisk(),
- n.getCanonicalFlavor(),
- n.getMembership().clusterid,
- clusterTypeOf(n.getMembership().clustertype)))
- .collect(Collectors.toUnmodifiableList());
+ .map(NodeRepository::toNode)
+ .collect(Collectors.toUnmodifiableList());
}
/** List all nodes in states, in zone owned by given application */
@@ -79,9 +67,35 @@ public interface NodeRepository {
/** Cancels firmware checks on all hosts in the given zone. */
void cancelFirmwareCheck(ZoneId zone);
+ private static Node toNode(NodeRepositoryNode node) {
+ var application = Optional.ofNullable(node.getOwner())
+ .map(owner -> ApplicationId.from(owner.getTenant(), owner.getApplication(),
+ owner.getInstance()));
+ return new Node(com.yahoo.config.provision.HostName.from(node.getHostname()),
+ fromJacksonState(node.getState()),
+ fromJacksonType(node.getType()),
+ application,
+ Version.fromString(node.getVespaVersion()),
+ Version.fromString(node.getWantedVespaVersion()),
+ Version.fromString(node.getCurrentOsVersion()),
+ Version.fromString(node.getWantedOsVersion()),
+ fromBoolean(node.getAllowedToBeDown()),
+ node.getCurrentRestartGeneration(),
+ node.getRestartGeneration(),
+ node.getCurrentRebootGeneration(),
+ node.getRebootGeneration(),
+ node.getMinCpuCores(),
+ node.getMinMainMemoryAvailableGb(),
+ node.getMinDiskAvailableGb(),
+ node.getBandwidth() / 1000,
+ node.getFastDisk(),
+ node.getCost() == null ? 0 : node.getCost(),
+ node.getCanonicalFlavor(),
+ node.getMembership().clusterid,
+ clusterTypeOf(node.getMembership().clustertype));
+ }
-
- private Node.ClusterType clusterTypeOf(String type) {
+ private static Node.ClusterType clusterTypeOf(String type) {
switch (type) {
case "admin": return Node.ClusterType.admin;
case "content": return Node.ClusterType.content;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java
index a6f47e34170..e30d2d55f77 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java
@@ -1,17 +1,15 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.api.integration.resource;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
-
-import java.util.List;
-
/**
- * Stores the total amount of resources allocated to a list of nodes
+ * An allocation of node resources.
*
- * @author leandroalves
+ * @author ldalves
*/
public class ResourceAllocation {
+ public static final ResourceAllocation ZERO = new ResourceAllocation(0, 0, 0);
+
private final double cpuCores;
private final double memoryGb;
private final double diskGb;
@@ -22,14 +20,6 @@ public class ResourceAllocation {
this.diskGb = diskGb;
}
- public static ResourceAllocation from(List<NodeRepositoryNode> nodes) {
- return new ResourceAllocation(
- nodes.stream().mapToDouble(NodeRepositoryNode::getMinCpuCores).sum(),
- nodes.stream().mapToDouble(NodeRepositoryNode::getMinMainMemoryAvailableGb).sum(),
- nodes.stream().mapToDouble(NodeRepositoryNode::getMinDiskAvailableGb).sum()
- );
- }
-
public double usageFraction(ResourceAllocation total) {
return (cpuCores / total.cpuCores + memoryGb / total.memoryGb + diskGb / total.diskGb) / 3;
}
@@ -46,5 +36,10 @@ public class ResourceAllocation {
return diskGb;
}
+ /** Returns a copy of this with the given allocation added */
+ public ResourceAllocation plus(ResourceAllocation allocation) {
+ return new ResourceAllocation(cpuCores + allocation.cpuCores, memoryGb + allocation.memoryGb, diskGb + allocation.diskGb);
+ }
+
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java
index bd4d31e53ae..a378bcb63bd 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java
@@ -2,7 +2,7 @@
package com.yahoo.vespa.hosted.controller.api.integration.resource;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
import java.time.Instant;
import java.util.List;
@@ -24,18 +24,19 @@ public class ResourceSnapshot {
this.timestamp = timestamp;
}
- public static ResourceSnapshot from(List<NodeRepositoryNode> nodes, Instant timestamp) {
+ public static ResourceSnapshot from(List<Node> nodes, Instant timestamp) {
Set<ApplicationId> applicationIds = nodes.stream()
- .map(n -> ApplicationId.from(n.getOwner().tenant, n.getOwner().application, n.getOwner().instance))
- .collect(Collectors.toSet());
+ .filter(node -> node.owner().isPresent())
+ .map(node -> node.owner().get())
+ .collect(Collectors.toSet());
if (applicationIds.size() != 1) throw new IllegalArgumentException("List of nodes can only represent one application");
return new ResourceSnapshot(
applicationIds.iterator().next(),
- nodes.stream().mapToDouble(NodeRepositoryNode::getMinCpuCores).sum(),
- nodes.stream().mapToDouble(NodeRepositoryNode::getMinMainMemoryAvailableGb).sum(),
- nodes.stream().mapToDouble(NodeRepositoryNode::getMinDiskAvailableGb).sum(),
+ nodes.stream().mapToDouble(Node::vcpu).sum(),
+ nodes.stream().mapToDouble(Node::memoryGb).sum(),
+ nodes.stream().mapToDouble(Node::diskGb).sum(),
timestamp
);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
index d9aaef6ef3b..4112d04d627 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
@@ -2,11 +2,12 @@
package com.yahoo.vespa.hosted.controller.maintenance;
import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList;
import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
import com.yahoo.vespa.hosted.controller.application.Deployment;
@@ -33,43 +34,37 @@ public class ClusterInfoMaintainer extends Maintainer {
private final Controller controller;
private final NodeRepository nodeRepository;
- ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl,
- NodeRepository nodeRepository) {
+ ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl) {
super(controller, duration, jobControl);
this.controller = controller;
- this.nodeRepository = nodeRepository;
+ this.nodeRepository = controller.configServer().nodeRepository();
}
private static String clusterId(NodeRepositoryNode node) {
return node.getMembership().clusterid;
}
- private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeList nodes) {
+ private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(List<Node> nodes) {
Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>();
// Group nodes by clusterid
- Map<String, List<NodeRepositoryNode>> clusters = nodes.nodes().stream()
- .filter(node -> node.getMembership() != null)
- .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterId));
+ Map<String, List<Node>> clusters = nodes.stream().collect(Collectors.groupingBy(Node::clusterId));
// For each cluster - get info
for (String id : clusters.keySet()) {
- List<NodeRepositoryNode> clusterNodes = clusters.get(id);
+ List<Node> clusterNodes = clusters.get(id);
// Assume they are all equal and use first node as a representative for the cluster
- NodeRepositoryNode node = clusterNodes.get(0);
-
- // Extract flavor info
- double cpu = 0;
- double mem = 0;
- double disk = 0;
+ Node node = clusterNodes.get(0);
// Add to map
- List<String> hostnames = clusterNodes.stream().map(NodeRepositoryNode::getHostname).collect(Collectors.toList());
- int cost = node.getCost() == null ? 0 : node.getCost(); // Cost is not guaranteed to be defined for all flavors
- ClusterInfo inf = new ClusterInfo(node.getFlavor(), cost, cpu, mem, disk,
- ClusterSpec.Type.from(node.getMembership().clustertype), hostnames);
- infoMap.put(new ClusterSpec.Id(id), inf);
+ List<String> hostnames = clusterNodes.stream()
+ .map(Node::hostname)
+ .map(HostName::value)
+ .collect(Collectors.toList());
+ ClusterInfo info = new ClusterInfo(node.canonicalFlavor(), node.cost(), node.vcpu(), node.memoryGb(), node.diskGb(),
+ ClusterSpec.Type.from(node.clusterType().name()), hostnames);
+ infoMap.put(new ClusterSpec.Id(id), info);
}
return infoMap;
@@ -81,7 +76,7 @@ public class ClusterInfoMaintainer extends Maintainer {
for (Deployment deployment : application.deployments().values()) {
DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone());
try {
- NodeList nodes = nodeRepository.listNodes(deploymentId.zoneId(), deploymentId.applicationId());
+ var nodes = nodeRepository.list(deploymentId.zoneId(), deploymentId.applicationId());
Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes);
controller().applications().lockIfPresent(application.id(), lockedApplication ->
controller.applications().store(lockedApplication.withClusterInfo(deployment.zone(), clusterInfo)));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index ca3fa2554c7..4ef8576066e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -6,9 +6,12 @@ import com.yahoo.config.provision.zone.ZoneApi;
import com.yahoo.jdisc.Metric;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.*;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandler;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
@@ -16,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer;
import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
-import java.time.Clock;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
@@ -60,7 +62,7 @@ public class ControllerMaintenance extends AbstractComponent {
public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator,
JobControl jobControl, Metric metric,
DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues,
- NameService nameService, NodeRepository nodeRepository,
+ NameService nameService,
ContactRetriever contactRetriever,
CostReportConsumer reportConsumer,
MeteringClient meteringClient,
@@ -77,7 +79,7 @@ public class ControllerMaintenance extends AbstractComponent {
versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(1), jobControl);
upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator);
readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofMinutes(1), jobControl);
- clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepository);
+ clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl);
clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl);
deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl);
applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues);
@@ -86,9 +88,9 @@ public class ControllerMaintenance extends AbstractComponent {
osUpgraders = osUpgraders(controller, jobControl);
osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl);
contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever);
- costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepository, Clock.systemUTC(), selfHostedCostConfig);
- resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(30), jobControl, nodeRepository, Clock.systemUTC(), metric, meteringClient);
nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService);
+ costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, selfHostedCostConfig);
+ resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(30), jobControl, metric, meteringClient);
billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billing);
awsEventReporterMaintainer = new AwsEventReporterMaintainer(controller, Duration.ofDays(1), jobControl, issueHandler, awsEventFetcher);
rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval, jobControl);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
index f3dac2be22e..436cb7e634d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
@@ -13,8 +13,6 @@ import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfi
import java.time.Clock;
import java.time.Duration;
import java.util.EnumSet;
-import java.util.Objects;
-import java.util.logging.Logger;
/**
* Periodically calculate and store cost allocation for properties.
@@ -24,8 +22,6 @@ import java.util.logging.Logger;
*/
public class CostReportMaintainer extends Maintainer {
- private static final Logger log = Logger.getLogger(CostReportMaintainer.class.getName());
-
private final CostReportConsumer consumer;
private final NodeRepository nodeRepository;
private final Clock clock;
@@ -36,13 +32,11 @@ public class CostReportMaintainer extends Maintainer {
public CostReportMaintainer(Controller controller, Duration interval,
CostReportConsumer consumer,
JobControl jobControl,
- NodeRepository nodeRepository,
- Clock clock,
SelfHostedCostConfig selfHostedCostConfig) {
super(controller, interval, jobControl, "CostReportMaintainer", EnumSet.of(SystemName.main));
this.consumer = consumer;
- this.nodeRepository = Objects.requireNonNull(nodeRepository, "node repository must be non-null");
- this.clock = clock;
+ this.nodeRepository = controller.configServer().nodeRepository();
+ this.clock = controller.clock();
this.selfHostedCostConfig = selfHostedCostConfig;
}
@@ -50,4 +44,5 @@ public class CostReportMaintainer extends Maintainer {
protected void maintain() {
consumer.Consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, selfHostedCostConfig, CloudName.from("yahoo")));
}
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
index c63dd28c13d..703cec348a8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
@@ -1,20 +1,18 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.maintenance;
-import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.SystemName;
import com.yahoo.jdisc.Metric;
import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
import java.time.Clock;
import java.time.Duration;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@@ -38,20 +36,18 @@ public class ResourceMeterMaintainer extends Maintainer {
public ResourceMeterMaintainer(Controller controller,
Duration interval,
JobControl jobControl,
- NodeRepository nodeRepository,
- Clock clock,
Metric metric,
MeteringClient meteringClient) {
super(controller, interval, jobControl, null, SystemName.all());
- this.clock = clock;
- this.nodeRepository = nodeRepository;
+ this.clock = controller.clock();
+ this.nodeRepository = controller.configServer().nodeRepository();
this.metric = metric;
this.meteringClient = meteringClient;
}
@Override
protected void maintain() {
- List<NodeRepositoryNode> nodes = getNodes();
+ List<Node> nodes = getNodes();
List<ResourceSnapshot> resourceSnapshots = getResourceSnapshots(nodes);
meteringClient.consume(resourceSnapshots);
@@ -63,26 +59,22 @@ public class ResourceMeterMaintainer extends Maintainer {
, metric.createContext(Collections.emptyMap()));
}
- private List<NodeRepositoryNode> getNodes() {
+ private List<Node> getNodes() {
return controller().zoneRegistry().zones()
.ofCloud(CloudName.from("aws"))
.reachable().zones().stream()
- .flatMap(zone -> nodeRepository.listNodes(zone.getId()).nodes().stream())
- .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa"))
- .filter(node -> node.getState() == NodeState.active)
+ .flatMap(zone -> nodeRepository.list(zone.getId()).stream())
+ .filter(node -> node.owner().isPresent() && !node.owner().get().tenant().value().equals("hosted-vespa"))
+ .filter(node -> node.state() == Node.State.active)
.collect(Collectors.toList());
}
- private List<ResourceSnapshot> getResourceSnapshots(List<NodeRepositoryNode> nodes) {
- return nodes.stream()
- .collect(Collectors.groupingBy(
- node -> applicationIdFromNodeOwner(node.getOwner()),
- Collectors.collectingAndThen(Collectors.toList(), nodeList -> ResourceSnapshot.from(nodeList, clock.instant()))
- )).values().stream().collect(Collectors.toList());
- }
-
- private ApplicationId applicationIdFromNodeOwner(NodeOwner owner) {
- return ApplicationId.from(owner.getTenant(), owner.getApplication(), owner.getInstance());
+ private List<ResourceSnapshot> getResourceSnapshots(List<Node> nodes) {
+ return new ArrayList<>(nodes.stream()
+ .filter(node -> node.owner().isPresent())
+ .collect(Collectors.groupingBy(node -> node.owner().get(),
+ Collectors.collectingAndThen(Collectors.toList(), nodeList -> ResourceSnapshot.from(nodeList, clock.instant()))
+ )).values());
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
index 796f786d823..bdaa86a59ff 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
@@ -16,16 +16,19 @@ import java.util.Optional;
import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
+/**
+ * @author ldalves
+ */
public class CostApiHandler extends LoggingRequestHandler {
private final Controller controller;
private final NodeRepository nodeRepository;
private final SelfHostedCostConfig selfHostedCostConfig;
- public CostApiHandler(Context ctx, Controller controller, NodeRepository nodeRepository, SelfHostedCostConfig selfHostedCostConfig) {
+ public CostApiHandler(Context ctx, Controller controller, SelfHostedCostConfig selfHostedCostConfig) {
super(ctx);
this.controller = controller;
- this.nodeRepository = nodeRepository;
+ this.nodeRepository = controller.configServer().nodeRepository();
this.selfHostedCostConfig = selfHostedCostConfig;
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
index 919cade1b05..50c7732bf76 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
@@ -2,25 +2,29 @@ package com.yahoo.vespa.hosted.controller.restapi.cost;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
+import com.yahoo.vespa.hosted.controller.tenant.Tenant;
+import java.text.SimpleDateFormat;
import java.time.Clock;
-import java.time.LocalDate;
import java.util.Comparator;
-import java.util.List;
+import java.util.Date;
+import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.yahoo.yolean.Exceptions.uncheck;
+/**
+ * @author ldalves
+ */
public class CostCalculator {
private static final double SELF_HOSTED_DISCOUNT = .5;
@@ -31,52 +35,45 @@ public class CostCalculator {
SelfHostedCostConfig selfHostedCostConfig,
CloudName cloudName) {
- String date = LocalDate.now(clock).toString();
-
- List<NodeRepositoryNode> nodes = controller.zoneRegistry().zones()
- .reachable().in(Environment.prod).ofCloud(cloudName).zones().stream()
- .flatMap(zone -> uncheck(() -> nodeRepository.listNodes(zone.getId()).nodes().stream()))
- .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa"))
- .collect(Collectors.toList());
-
- if (cloudName.equals(CloudName.from("yahoo")))
- selfHostedCostConfig.properties().stream().map(property -> {
- NodeRepositoryNode selfHostedNode = new NodeRepositoryNode();
-
- NodeOwner owner = new NodeOwner();
- owner.tenant = property.name();
- selfHostedNode.setOwner(owner);
- selfHostedNode.setMinCpuCores(property.cpuCores() * SELF_HOSTED_DISCOUNT);
- selfHostedNode.setMinMainMemoryAvailableGb(property.memoryGb() * SELF_HOSTED_DISCOUNT);
- selfHostedNode.setMinDiskAvailableGb(property.diskGb() * SELF_HOSTED_DISCOUNT);
-
- return selfHostedNode;
- }).forEach(nodes::add);
-
- ResourceAllocation totalResourceAllocation = ResourceAllocation.from(nodes);
-
- Map<String, Property> propertyByTenantName = controller.tenants().asList().stream()
- .filter(AthenzTenant.class::isInstance)
- .collect(Collectors.toMap(
- tenant -> tenant.name().value(),
- tenant -> ((AthenzTenant) tenant).property()
- ));
-
- selfHostedCostConfig.properties().stream()
- .map(SelfHostedCostConfig.Properties::name)
- .forEach(name -> propertyByTenantName.put(name, new Property(name)));
-
- Map<Property, ResourceAllocation> resourceShareByProperty = nodes.stream()
- .filter(node -> propertyByTenantName.containsKey(node.getOwner().tenant))
- .collect(Collectors.groupingBy(
- node -> propertyByTenantName.get(node.getOwner().tenant),
- Collectors.collectingAndThen(
- Collectors.toList(),
- ResourceAllocation::from
- )
- ));
-
- return toCsv(resourceShareByProperty, date, totalResourceAllocation);
+ var date = new SimpleDateFormat("yyyy-MM-dd").format(Date.from(clock.instant()));
+
+ // Group properties by tenant name
+ Map<TenantName, Property> propertyByTenantName = controller.tenants().asList().stream()
+ .filter(AthenzTenant.class::isInstance)
+ .collect(Collectors.toMap(Tenant::name,
+ tenant -> ((AthenzTenant) tenant).property()));
+
+ // Sum up allocations
+ Map<Property, ResourceAllocation> allocationByProperty = new HashMap<>();
+ var nodes = controller.zoneRegistry().zones()
+ .reachable().in(Environment.prod).ofCloud(cloudName).zones().stream()
+ .flatMap(zone -> uncheck(() -> nodeRepository.list(zone.getId()).stream()))
+ .filter(node -> node.owner().isPresent() && !node.owner().get().tenant().value().equals("hosted-vespa"))
+ .collect(Collectors.toList());
+ var totalAllocation = ResourceAllocation.ZERO;
+ for (var node : nodes) {
+ Property property = propertyByTenantName.get(node.owner().get().tenant());
+ if (property == null) continue;
+ var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO);
+ var nodeAllocation = new ResourceAllocation(node.vcpu(), node.memoryGb(), node.diskGb());
+ allocationByProperty.put(property, allocation.plus(nodeAllocation));
+ totalAllocation = totalAllocation.plus(nodeAllocation);
+ }
+
+ // Add fixed allocations from config
+ if (cloudName.equals(CloudName.from("yahoo"))) {
+ for (var propertyEntry : selfHostedCostConfig.properties()) {
+ var property = new Property(propertyEntry.name());
+ var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO);
+ var fixedAllocation = new ResourceAllocation(propertyEntry.cpuCores() * SELF_HOSTED_DISCOUNT,
+ propertyEntry.memoryGb() * SELF_HOSTED_DISCOUNT,
+ propertyEntry.diskGb() * SELF_HOSTED_DISCOUNT);
+ allocationByProperty.put(property, allocation.plus(fixedAllocation));
+ totalAllocation = totalAllocation.plus(fixedAllocation);
+ }
+ }
+
+ return toCsv(allocationByProperty, date, totalAllocation);
}
private static String toCsv(Map<Property, ResourceAllocation> resourceShareByProperty, String date, ResourceAllocation totalResourceAllocation) {
@@ -91,4 +88,5 @@ public class CostCalculator {
.collect(Collectors.joining("\n"));
return header + entries;
}
+
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java
deleted file mode 100644
index 4f2d85adfbe..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java
+++ /dev/null
@@ -1,119 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.integration;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
-import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState;
-
-import java.util.Collection;
-import java.util.List;
-
-/**
- * @author bjorncs
- */
-public class NodeRepositoryClientMock implements NodeRepository {
-
- @Override
- public void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public NodeRepositoryNode getNode(ZoneId zone, String hostname) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void deleteNode(ZoneId zone, String hostname) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public NodeList listNodes(ZoneId zone) {
- NodeRepositoryNode nodeA = createNodeA();
- NodeRepositoryNode nodeB = createNodeB();
- return new NodeList(List.of(nodeA, nodeB));
- }
-
- @Override
- public NodeList listNodes(ZoneId zone, ApplicationId application) {
- NodeRepositoryNode nodeA = createNodeA();
- NodeRepositoryNode nodeB = createNodeB();
- return new NodeList(List.of(nodeA, nodeB));
- }
-
- private static NodeRepositoryNode createNodeA() {
- NodeRepositoryNode node = new NodeRepositoryNode();
- node.setHostname("hostA");
- node.setCost(10);
- node.setFlavor("C-2B/24/500");
- node.setMinCpuCores(24d);
- node.setMinDiskAvailableGb(500d);
- node.setMinMainMemoryAvailableGb(24d);
- NodeOwner owner = new NodeOwner();
- owner.tenant = "tenant1";
- owner.application = "app1";
- owner.instance = "default";
- node.setOwner(owner);
- NodeMembership membership = new NodeMembership();
- membership.clusterid = "clusterA";
- membership.clustertype = "container";
- node.setMembership(membership);
- node.setState(NodeState.active);
- return node;
- }
-
- private static NodeRepositoryNode createNodeB() {
- NodeRepositoryNode node = new NodeRepositoryNode();
- node.setHostname("hostB");
- node.setCost(20);
- node.setFlavor("C-2C/24/500");
- node.setMinCpuCores(40d);
- node.setMinDiskAvailableGb(500d);
- node.setMinMainMemoryAvailableGb(24d);
- NodeOwner owner = new NodeOwner();
- owner.tenant = "tenant2";
- owner.application = "app2";
- owner.instance = "default";
- node.setOwner(owner);
- NodeMembership membership = new NodeMembership();
- membership.clusterid = "clusterB";
- membership.clustertype = "content";
- node.setMembership(membership);
- node.setState(NodeState.active);
- return node;
- }
-
- @Override
- public void setState(ZoneId zone, NodeState nodeState, String nodename) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void upgrade(ZoneId zone, NodeType type, Version version) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void upgradeOs(ZoneId zone, NodeType type, Version version) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void requestFirmwareCheck(ZoneId zone) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void cancelFirmwareCheck(ZoneId zone) {
- throw new UnsupportedOperationException();
- }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
index 61cb5c4a2ed..687fc9f8b29 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
@@ -59,6 +59,58 @@ public class NodeRepositoryMock implements NodeRepository {
.orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known."));
}
+ public void addNodes(ZoneId zone, List<Node> nodes) {
+ nodeRepository.put(zone, nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity())));
+ }
+
+ public void addFixedNodes(ZoneId zone) {
+ var nodeA = new Node(HostName.from("hostA"),
+ Node.State.active,
+ NodeType.tenant,
+ Optional.of(ApplicationId.from("tenant1", "app1", "default")),
+ Version.fromString("7.42"),
+ Version.fromString("7.42"),
+ Version.fromString("7.6"),
+ Version.fromString("7.6"),
+ Node.ServiceState.expectedUp,
+ 0,
+ 0,
+ 0,
+ 0,
+ 24,
+ 24,
+ 500,
+ 1000,
+ false,
+ 10,
+ "C-2B/24/500",
+ "clusterA",
+ Node.ClusterType.container);
+ var nodeB = new Node(HostName.from("hostB"),
+ Node.State.active,
+ NodeType.tenant,
+ Optional.of(ApplicationId.from("tenant2", "app2", "default")),
+ Version.fromString("7.42"),
+ Version.fromString("7.42"),
+ Version.fromString("7.6"),
+ Version.fromString("7.6"),
+ Node.ServiceState.expectedUp,
+ 0,
+ 0,
+ 0,
+ 0,
+ 40,
+ 24,
+ 500,
+ 1000,
+ false,
+ 20,
+ "C-2C/24/500",
+ "clusterB",
+ Node.ClusterType.container);
+ addNodes(zone, List.of(nodeA, nodeB));
+ }
+
@Override
public void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) {
throw new UnsupportedOperationException();
@@ -90,6 +142,11 @@ public class NodeRepositoryMock implements NodeRepository {
}
@Override
+ public List<Node> list(ZoneId zone) {
+ return List.copyOf(nodeRepository.getOrDefault(zone, Map.of()).values());
+ }
+
+ @Override
public List<Node> list(ZoneId zone, ApplicationId application) {
return nodeRepository.getOrDefault(zone, Collections.emptyMap()).values().stream()
.filter(node -> node.owner().map(application::equals).orElse(false))
@@ -129,6 +186,7 @@ public class NodeRepositoryMock implements NodeRepository {
node.diskGb(),
node.bandwidthGbps(),
node.fastDisk(),
+ node.cost(),
node.canonicalFlavor(),
node.clusterId(),
node.clusterType()))
@@ -137,12 +195,10 @@ public class NodeRepositoryMock implements NodeRepository {
@Override
public void requestFirmwareCheck(ZoneId zone) {
- ;
}
@Override
public void cancelFirmwareCheck(ZoneId zone) {
- ;
}
public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) {
@@ -179,6 +235,7 @@ public class NodeRepositoryMock implements NodeRepository {
node.diskGb(),
node.bandwidthGbps(),
node.fastDisk(),
+ node.cost(),
node.canonicalFlavor(),
node.clusterId(),
node.clusterType()));
@@ -203,6 +260,7 @@ public class NodeRepositoryMock implements NodeRepository {
node.diskGb(),
node.bandwidthGbps(),
node.fastDisk(),
+ node.cost(),
node.canonicalFlavor(),
node.clusterId(),
node.clusterType()));
@@ -227,6 +285,7 @@ public class NodeRepositoryMock implements NodeRepository {
node.diskGb(),
node.bandwidthGbps(),
node.fastDisk(),
+ node.cost(),
node.canonicalFlavor(),
node.clusterId(),
node.clusterType()));
@@ -251,6 +310,7 @@ public class NodeRepositoryMock implements NodeRepository {
node.diskGb(),
node.bandwidthGbps(),
node.fastDisk(),
+ node.cost(),
node.canonicalFlavor(),
node.clusterId(),
node.clusterType()));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java
index ec504b1c6dd..72b16d76864 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java
@@ -1,16 +1,22 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.maintenance;
+import com.yahoo.component.Version;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.ControllerTester;
-import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
import org.junit.Test;
import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
import static org.junit.Assert.assertEquals;
@@ -19,9 +25,10 @@ import static org.junit.Assert.assertEquals;
*/
public class ClusterInfoMaintainerTest {
+ private final ControllerTester tester = new ControllerTester();
+
@Test
public void maintain() {
- ControllerTester tester = new ControllerTester();
ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1",
Environment.dev, 123).id();
@@ -31,9 +38,9 @@ public class ClusterInfoMaintainerTest {
.get();
assertEquals(0, deployment.clusterInfo().size());
+ addNodes(ZoneId.from("dev", "us-east-1"));
ClusterInfoMaintainer maintainer = new ClusterInfoMaintainer(tester.controller(), Duration.ofHours(1),
- new JobControl(new MockCuratorDb()),
- new NodeRepositoryClientMock());
+ new JobControl(new MockCuratorDb()));
maintainer.maintain();
deployment = tester.controller().applications().get(app).get().deployments().values().stream()
@@ -43,4 +50,52 @@ public class ClusterInfoMaintainerTest {
assertEquals(10, deployment.clusterInfo().get(ClusterSpec.Id.from("clusterA")).getFlavorCost());
}
+ private void addNodes(ZoneId zone) {
+ var nodeA = new Node(HostName.from("hostA"),
+ Node.State.active,
+ NodeType.tenant,
+ Optional.of(ApplicationId.from("tenant1", "app1", "default")),
+ Version.fromString("7.42"),
+ Version.fromString("7.42"),
+ Version.fromString("7.6"),
+ Version.fromString("7.6"),
+ Node.ServiceState.expectedUp,
+ 0,
+ 0,
+ 0,
+ 0,
+ 24,
+ 24,
+ 500,
+ 1000,
+ false,
+ 10,
+ "C-2B/24/500",
+ "clusterA",
+ Node.ClusterType.container);
+ var nodeB = new Node(HostName.from("hostB"),
+ Node.State.active,
+ NodeType.tenant,
+ Optional.of(ApplicationId.from("tenant1", "app1", "default")),
+ Version.fromString("7.42"),
+ Version.fromString("7.42"),
+ Version.fromString("7.6"),
+ Version.fromString("7.6"),
+ Node.ServiceState.expectedUp,
+ 0,
+ 0,
+ 0,
+ 0,
+ 40,
+ 24,
+ 500,
+ 1000,
+ false,
+ 20,
+ "C-2C/24/500",
+ "clusterB",
+ Node.ClusterType.container);
+ tester.configServer().nodeRepository().addNodes(zone, List.of(nodeA, nodeB));
+ }
+
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
index 1edf371924a..892da281667 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
@@ -1,36 +1,39 @@
package com.yahoo.vespa.hosted.controller.maintenance;
-import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.ControllerTester;
-import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock;
import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer;
import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
-import org.junit.Assert;
import org.junit.Test;
-import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
-import java.util.ArrayList;
-import java.util.List;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author ldalves
+ */
public class CostReportMaintainerTest {
+ private final ControllerTester tester = new ControllerTester();
+
@Test
public void maintain() {
- ControllerTester tester = new ControllerTester();
+ tester.clock().setInstant(Instant.EPOCH);
tester.zoneRegistry().setZones(
ZoneApiMock.newBuilder().withId("prod.us-east-3").withCloud("yahoo").build(),
ZoneApiMock.newBuilder().withId("prod.us-west-1").withCloud("yahoo").build(),
ZoneApiMock.newBuilder().withId("prod.us-central-1").withCloud("yahoo").build(),
ZoneApiMock.newBuilder().withId("prod.eu-west-1").withCloud("yahoo").build());
+ addNodes();
- CostReportConsumer mockConsumer = csv -> Assert.assertEquals(csv,
+ CostReportConsumer mockConsumer = csv -> assertEquals(
"Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n" +
"1970-01-01,Property1,96.0,96.0,2000.0,0.3055555555555555\n" +
"1970-01-01,Property3,128.0,96.0,2000.0,0.3333333333333333\n" +
- "1970-01-01,Property2,160.0,96.0,2000.0,0.3611111111111111");
+ "1970-01-01,Property2,160.0,96.0,2000.0,0.3611111111111111",
+ csv);
SelfHostedCostConfig costConfig = new SelfHostedCostConfig.Builder()
.properties(
@@ -49,9 +52,14 @@ public class CostReportMaintainerTest {
Duration.ofDays(1),
mockConsumer,
new JobControl(tester.curator()),
- new NodeRepositoryClientMock(),
- Clock.fixed(Instant.EPOCH, java.time.ZoneId.of("UTC")),
costConfig);
maintainer.maintain();
}
-} \ No newline at end of file
+
+ private void addNodes() {
+ for (var zone : tester.zoneRegistry().zones().all().zones()) {
+ tester.configServer().nodeRepository().addFixedNodes(zone.getId());
+ }
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
index 4bee9fe2f75..dedb3a4cc17 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
@@ -165,7 +165,8 @@ public class OsUpgraderTest {
node.wantedVersion(), node.wantedOsVersion(), node.wantedOsVersion(), node.serviceState(),
node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(),
node.wantedRebootGeneration(), node.vcpu(), node.memoryGb(), node.diskGb(),
- node.bandwidthGbps(), node.fastDisk(), node.canonicalFlavor(), node.clusterId(), node.clusterType()));
+ node.bandwidthGbps(), node.fastDisk(), node.cost(), node.canonicalFlavor(),
+ node.clusterId(), node.clusterType()));
}
assertCurrent(version, application, zone);
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
index 76568ef17e0..d8d7995f382 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
@@ -1,11 +1,10 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.maintenance;
-import com.yahoo.vespa.hosted.controller.ControllerTester;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient;
-import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock;
import com.yahoo.vespa.hosted.controller.integration.MetricsMock;
import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
import org.junit.Test;
@@ -13,28 +12,29 @@ import org.junit.Test;
import java.time.Duration;
import java.util.List;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
/**
* @author olaa
*/
public class ResourceMeterMaintainerTest {
+ private final ControllerTester tester = new ControllerTester();
private final double DELTA = Double.MIN_VALUE;
- private NodeRepositoryClientMock nodeRepository = new NodeRepositoryClientMock();
private MockMeteringClient snapshotConsumer = new MockMeteringClient();
private MetricsMock metrics = new MetricsMock();
@Test
public void testMaintainer() {
- ControllerTester tester = new ControllerTester();
+ var awsZone = ZoneApiMock.newBuilder().withId("prod.aws-us-east-1").withCloud("aws").build();
tester.zoneRegistry().setZones(
ZoneApiMock.newBuilder().withId("prod.us-east-3").build(),
ZoneApiMock.newBuilder().withId("prod.us-west-1").build(),
ZoneApiMock.newBuilder().withId("prod.us-central-1").build(),
- ZoneApiMock.newBuilder().withId("prod.aws-us-east-1").withCloud("aws").build());
+ awsZone);
+ tester.configServer().nodeRepository().addFixedNodes(awsZone.getId());
- ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), nodeRepository, tester.clock(), metrics, snapshotConsumer);
+ ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), metrics, snapshotConsumer);
resourceMeterMaintainer.maintain();
List<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources();
@@ -54,4 +54,5 @@ public class ResourceMeterMaintainerTest {
assertEquals(tester.clock().millis()/1000, metrics.getMetric("metering_last_reported"));
assertEquals(1112.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA);
}
-} \ No newline at end of file
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index f2fbf36bec2..7be0be9c80d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -77,7 +77,6 @@ public class ControllerContainerTest {
" <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockBilling'/>\n" +
" <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient'/>\n" +
" <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerMock'/>\n" +
- " <component id='com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock'/>\n" +
" <component id='com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock'/>\n" +
" <component id='com.yahoo.vespa.hosted.controller.Controller'/>\n" +
" <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService'/>\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
index 653ab38dc3a..3ef6dfc403d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
@@ -150,7 +150,8 @@ public class OsApiTest extends ControllerContainerTest {
node.wantedVersion(), node.wantedOsVersion(), node.wantedOsVersion(), node.serviceState(),
node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(),
node.wantedRebootGeneration(), node.vcpu(), node.memoryGb(), node.diskGb(),
- node.bandwidthGbps(), node.fastDisk(), node.canonicalFlavor(), node.clusterId(), node.clusterType()));
+ node.bandwidthGbps(), node.fastDisk(), node.cost(), node.canonicalFlavor(),
+ node.clusterId(), node.clusterType()));
}
}
}