diff options
author | Leandro Alves <ldalves@gmail.com> | 2018-12-13 10:34:33 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-12-13 10:34:33 +0100 |
commit | 6e9384c64f0325f45619ba9ac9e057afdf18794e (patch) | |
tree | 5cbee91269b772fa451ae79980ef5149b483ea54 | |
parent | dcf10d9fd1f13dfcb1a5691a3865c62e007fa8ed (diff) | |
parent | a2bec45d9b2b9ccc8a1c3a9a5b2bc44db58c8d53 (diff) |
Merge pull request #7929 from vespa-engine/andreer/self-hosted-cost-config
configure cost for self-hosted properties
7 files changed, 72 insertions, 11 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index fafd27aaded..7924ea34475 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; +import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import java.time.Clock; import java.time.Duration; @@ -58,7 +59,8 @@ public class ControllerMaintenance extends AbstractComponent { DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient, ContactRetriever contactRetriever, - CostReportConsumer reportConsumer) { + CostReportConsumer reportConsumer, + SelfHostedCostConfig selfHostedCostConfig) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); this.jobControl = jobControl; deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl); @@ -78,7 +80,7 @@ public class ControllerMaintenance extends AbstractComponent { osUpgraders = osUpgraders(controller, jobControl); osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever); - costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC()); + costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 77febb71ca6..2298c3c92fe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; +import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import java.time.Clock; import java.time.Duration; @@ -26,6 +27,7 @@ public class CostReportMaintainer extends Maintainer { private final CostReportConsumer consumer; private final NodeRepositoryClientInterface nodeRepository; private final Clock clock; + private final SelfHostedCostConfig selfHostedCostConfig; @Inject @SuppressWarnings("WeakerAccess") @@ -33,15 +35,17 @@ public class CostReportMaintainer extends Maintainer { CostReportConsumer consumer, JobControl jobControl, NodeRepositoryClientInterface nodeRepository, - Clock clock) { + Clock clock, + SelfHostedCostConfig selfHostedCostConfig) { super(controller, interval, jobControl, "CostReportMaintainer", EnumSet.of(SystemName.main)); this.consumer = consumer; this.nodeRepository = Objects.requireNonNull(nodeRepository, "node repository must be non-null"); this.clock = clock; + this.selfHostedCostConfig = selfHostedCostConfig; } @Override protected void maintain() { - consumer.Consume(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller(), clock))); + consumer.Consume(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller(), clock, selfHostedCostConfig))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java index cdec0f8da74..6d599d32cc6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.StringResponse; +import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import java.time.Clock; @@ -17,11 +18,13 @@ public class CostApiHandler extends LoggingRequestHandler { private final Controller controller; private final NodeRepositoryClientInterface nodeRepository; + private final SelfHostedCostConfig selfHostedCostConfig; - public CostApiHandler(Context ctx, Controller controller, NodeRepositoryClientInterface nodeRepository) { + public CostApiHandler(Context ctx, Controller controller, NodeRepositoryClientInterface nodeRepository, SelfHostedCostConfig selfHostedCostConfig) { super(ctx); this.controller = controller; this.nodeRepository = nodeRepository; + this.selfHostedCostConfig = selfHostedCostConfig; } @Override @@ -33,7 +36,7 @@ public class CostApiHandler extends LoggingRequestHandler { Path path = new Path(request.getUri().getPath()); if (path.matches("/cost/v1/csv")) { - return new StringResponse(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller, Clock.systemUTC()))); + return new StringResponse(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller, Clock.systemUTC(), selfHostedCostConfig))); } return ErrorResponse.notFoundError("Nothing at " + path); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java index d6e77e6d381..d1774517099 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.controller.restapi.cost; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName; +import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import java.time.Clock; @@ -19,7 +21,13 @@ import java.util.stream.Stream; import static com.yahoo.yolean.Exceptions.uncheck; public class CostCalculator { - public static Map<Property, ResourceAllocation> calculateCost(NodeRepositoryClientInterface nodeRepository, Controller controller, Clock clock) { + + private static final double SELF_HOSTED_DISCOUNT = .5; + + public static Map<Property, ResourceAllocation> calculateCost(NodeRepositoryClientInterface nodeRepository, + Controller controller, + Clock clock, + SelfHostedCostConfig selfHostedCostConfig) { String date = LocalDate.now(clock).toString(); @@ -29,6 +37,19 @@ public class CostCalculator { .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) .collect(Collectors.toList()); + selfHostedCostConfig.properties().stream().map(property -> { + NodeRepositoryNode selfHostedNode = new NodeRepositoryNode(); + + NodeOwner owner = new NodeOwner(); + owner.tenant = property.name(); + selfHostedNode.setOwner(owner); + selfHostedNode.setMinCpuCores(property.cpuCores() * SELF_HOSTED_DISCOUNT); + selfHostedNode.setMinMainMemoryAvailableGb(property.memoryGb() * SELF_HOSTED_DISCOUNT); + selfHostedNode.setMinDiskAvailableGb(property.diskGb() * SELF_HOSTED_DISCOUNT); + + return selfHostedNode; + }).forEach(nodes::add); + ResourceAllocation total = ResourceAllocation.from(date, nodes, null); Map<String, Property> propertyByTenantName = controller.tenants().asList().stream() @@ -38,6 +59,10 @@ public class CostCalculator { tenant -> ((AthenzTenant) tenant).property() )); + selfHostedCostConfig.properties().stream() + .map(SelfHostedCostConfig.Properties::name) + .forEach(name -> propertyByTenantName.put(name, new Property(name))); + return nodes.stream() .filter(node -> propertyByTenantName.containsKey(node.getOwner().tenant)) .collect(Collectors.groupingBy( diff --git a/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def b/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def new file mode 100644 index 00000000000..059af1558a4 --- /dev/null +++ b/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def @@ -0,0 +1,7 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.controller.restapi.cost.config + +properties[].name string +properties[].cpuCores int +properties[].memoryGb int +properties[].diskGb int
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index 89370f0e863..01f3f55c679 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; +import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import org.junit.Assert; import org.junit.Test; @@ -19,12 +20,30 @@ public class CostReportMaintainerTest { CostReportConsumer mockConsumer = csv -> Assert.assertEquals(csv, "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n" + - "1970-01-01,Property1,96.0,96.0,2000.0,0.4583333333333333\n" + - "1970-01-01,Property2,160.0,96.0,2000.0,0.5416666666666666"); + "1970-01-01,Property1,96.0,96.0,2000.0,0.3055555555555555\n" + + "1970-01-01,Property3,128.0,96.0,2000.0,0.3333333333333333\n" + + "1970-01-01,Property2,160.0,96.0,2000.0,0.3611111111111111"); + + SelfHostedCostConfig costConfig = new SelfHostedCostConfig.Builder() + .properties( + new SelfHostedCostConfig.Properties.Builder() + .name("Property3") + .cpuCores(256) + .memoryGb(192) + .diskGb(4000)) + .build(); + tester.createTenant("lsbe", "local-search", 1L); tester.createTenant("mediasearch", "msbe", 2L); - CostReportMaintainer maintainer = new CostReportMaintainer(tester.controller(), Duration.ofDays(1), mockConsumer, new JobControl(tester.curator()), new NodeRepositoryClientMock(), Clock.fixed(Instant.EPOCH, ZoneId.of("UTC"))); + CostReportMaintainer maintainer = new CostReportMaintainer( + tester.controller(), + Duration.ofDays(1), + mockConsumer, + new JobControl(tester.curator()), + new NodeRepositoryClientMock(), + Clock.fixed(Instant.EPOCH, ZoneId.of("UTC")), + costConfig); maintainer.maintain(); } }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java index bc03c9e87b3..4b523446b13 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java @@ -37,7 +37,8 @@ public class CostApiTest extends ControllerContainerTest { @Test public void test_api() { - assertResponse(new Request("http://localhost:8080/cost/v1/csv"), "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200); + assertResponse(new Request("http://localhost:8080/cost/v1/csv"), + "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200); } private ZoneRegistryMock zoneRegistryMock() { |