Merge pull request #7929 from vespa-engine/andreer/self-hosted-cost-config

configure cost for self-hosted properties

7 files changed, 72 insertions, 11 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index fafd27aaded..7924ea34475 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer;
+import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -58,7 +59,8 @@ public class ControllerMaintenance extends AbstractComponent {
                                  DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues,
                                  NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient,
                                  ContactRetriever contactRetriever,
-                                 CostReportConsumer reportConsumer) {
+                                 CostReportConsumer reportConsumer,
+                                 SelfHostedCostConfig selfHostedCostConfig) {
         Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes());
         this.jobControl = jobControl;
         deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl);
@@ -78,7 +80,7 @@ public class ControllerMaintenance extends AbstractComponent {
         osUpgraders = osUpgraders(controller, jobControl);
         osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl);
         contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever);
-        costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC());
+        costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig);
     }
 
     public Upgrader upgrader() { return upgrader; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
index 77febb71ca6..2298c3c92fe 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
@@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
 import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator;
 import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer;
+import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -26,6 +27,7 @@ public class CostReportMaintainer extends Maintainer {
     private final CostReportConsumer consumer;
     private final NodeRepositoryClientInterface nodeRepository;
     private final Clock clock;
+    private final SelfHostedCostConfig selfHostedCostConfig;
 
     @Inject
     @SuppressWarnings("WeakerAccess")
@@ -33,15 +35,17 @@ public class CostReportMaintainer extends Maintainer {
                                 CostReportConsumer consumer,
                                 JobControl jobControl,
                                 NodeRepositoryClientInterface nodeRepository,
-                                Clock clock) {
+                                Clock clock,
+                                SelfHostedCostConfig selfHostedCostConfig) {
         super(controller, interval, jobControl, "CostReportMaintainer", EnumSet.of(SystemName.main));
         this.consumer = consumer;
         this.nodeRepository = Objects.requireNonNull(nodeRepository, "node repository must be non-null");
         this.clock = clock;
+        this.selfHostedCostConfig = selfHostedCostConfig;
     }
 
     @Override
     protected void maintain() {
-        consumer.Consume(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller(), clock)));
+        consumer.Consume(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller(), clock, selfHostedCostConfig)));
     }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
index cdec0f8da74..6d599d32cc6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.StringResponse;
+import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
 
 import java.time.Clock;
 
@@ -17,11 +18,13 @@ public class CostApiHandler extends LoggingRequestHandler {
 
     private final Controller controller;
     private final NodeRepositoryClientInterface nodeRepository;
+    private final SelfHostedCostConfig selfHostedCostConfig;
 
-    public CostApiHandler(Context ctx, Controller controller, NodeRepositoryClientInterface nodeRepository) {
+    public CostApiHandler(Context ctx, Controller controller, NodeRepositoryClientInterface nodeRepository, SelfHostedCostConfig selfHostedCostConfig) {
         super(ctx);
         this.controller = controller;
         this.nodeRepository = nodeRepository;
+        this.selfHostedCostConfig = selfHostedCostConfig;
     }
 
     @Override
@@ -33,7 +36,7 @@ public class CostApiHandler extends LoggingRequestHandler {
         Path path = new Path(request.getUri().getPath());
 
         if (path.matches("/cost/v1/csv")) {
-            return new StringResponse(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller, Clock.systemUTC())));
+            return new StringResponse(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller, Clock.systemUTC(), selfHostedCostConfig)));
         }
 
         return ErrorResponse.notFoundError("Nothing at " + path);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
index d6e77e6d381..d1774517099 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
@@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.controller.restapi.cost;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
+import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 
 import java.time.Clock;
@@ -19,7 +21,13 @@ import java.util.stream.Stream;
 import static com.yahoo.yolean.Exceptions.uncheck;
 
 public class CostCalculator {
-    public static Map<Property, ResourceAllocation> calculateCost(NodeRepositoryClientInterface nodeRepository, Controller controller, Clock clock) {
+
+    private static final double SELF_HOSTED_DISCOUNT = .5;
+
+    public static Map<Property, ResourceAllocation> calculateCost(NodeRepositoryClientInterface nodeRepository,
+                                                                  Controller controller,
+                                                                  Clock clock,
+                                                                  SelfHostedCostConfig selfHostedCostConfig) {
 
         String date = LocalDate.now(clock).toString();
 
@@ -29,6 +37,19 @@ public class CostCalculator {
                 .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa"))
                 .collect(Collectors.toList());
 
+        selfHostedCostConfig.properties().stream().map(property -> {
+            NodeRepositoryNode selfHostedNode = new NodeRepositoryNode();
+
+            NodeOwner owner = new NodeOwner();
+            owner.tenant = property.name();
+            selfHostedNode.setOwner(owner);
+            selfHostedNode.setMinCpuCores(property.cpuCores() * SELF_HOSTED_DISCOUNT);
+            selfHostedNode.setMinMainMemoryAvailableGb(property.memoryGb() * SELF_HOSTED_DISCOUNT);
+            selfHostedNode.setMinDiskAvailableGb(property.diskGb() * SELF_HOSTED_DISCOUNT);
+
+            return selfHostedNode;
+        }).forEach(nodes::add);
+
         ResourceAllocation total = ResourceAllocation.from(date, nodes, null);
 
         Map<String, Property> propertyByTenantName = controller.tenants().asList().stream()
@@ -38,6 +59,10 @@ public class CostCalculator {
                         tenant -> ((AthenzTenant) tenant).property()
                 ));
 
+        selfHostedCostConfig.properties().stream()
+                .map(SelfHostedCostConfig.Properties::name)
+                .forEach(name -> propertyByTenantName.put(name, new Property(name)));
+
         return nodes.stream()
                 .filter(node -> propertyByTenantName.containsKey(node.getOwner().tenant))
                 .collect(Collectors.groupingBy(
diff --git a/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def b/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def
new file mode 100644
index 00000000000..059af1558a4
--- /dev/null
+++ b/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def
@@ -0,0 +1,7 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=vespa.hosted.controller.restapi.cost.config
+
+properties[].name string
+properties[].cpuCores int
+properties[].memoryGb int
+properties[].diskGb int
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
index 89370f0e863..01f3f55c679 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock;
 import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer;
+import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -19,12 +20,30 @@ public class CostReportMaintainerTest {
 
         CostReportConsumer mockConsumer = csv -> Assert.assertEquals(csv,
                 "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n" +
-                        "1970-01-01,Property1,96.0,96.0,2000.0,0.4583333333333333\n" +
-                        "1970-01-01,Property2,160.0,96.0,2000.0,0.5416666666666666");
+                        "1970-01-01,Property1,96.0,96.0,2000.0,0.3055555555555555\n" +
+                        "1970-01-01,Property3,128.0,96.0,2000.0,0.3333333333333333\n" +
+                        "1970-01-01,Property2,160.0,96.0,2000.0,0.3611111111111111");
+
+        SelfHostedCostConfig costConfig = new SelfHostedCostConfig.Builder()
+                .properties(
+                        new SelfHostedCostConfig.Properties.Builder()
+                                .name("Property3")
+                                .cpuCores(256)
+                                .memoryGb(192)
+                                .diskGb(4000))
+                .build();
+
 
         tester.createTenant("lsbe", "local-search", 1L);
         tester.createTenant("mediasearch", "msbe", 2L);
-        CostReportMaintainer maintainer = new CostReportMaintainer(tester.controller(), Duration.ofDays(1), mockConsumer, new JobControl(tester.curator()), new NodeRepositoryClientMock(), Clock.fixed(Instant.EPOCH, ZoneId.of("UTC")));
+        CostReportMaintainer maintainer = new CostReportMaintainer(
+                tester.controller(),
+                Duration.ofDays(1),
+                mockConsumer,
+                new JobControl(tester.curator()),
+                new NodeRepositoryClientMock(),
+                Clock.fixed(Instant.EPOCH, ZoneId.of("UTC")),
+                costConfig);
         maintainer.maintain();
     }
 }
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java
index bc03c9e87b3..4b523446b13 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java
@@ -37,7 +37,8 @@ public class CostApiTest extends ControllerContainerTest {
 
     @Test
     public void test_api() {
-        assertResponse(new Request("http://localhost:8080/cost/v1/csv"), "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200);
+        assertResponse(new Request("http://localhost:8080/cost/v1/csv"),
+                "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200);
     }
 
     private ZoneRegistryMock zoneRegistryMock() {