diff options
14 files changed, 103 insertions, 77 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 73a4f56efcf..d74dbe13acb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandl import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; @@ -47,4 +48,6 @@ public interface ServiceRegistry { EntityService entityService(); + CostReportConsumer costReportConsumer(); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumer.java new file mode 100644 index 00000000000..5c4bfcf219d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostReportConsumer.java @@ -0,0 +1,16 @@ +package com.yahoo.vespa.hosted.controller.api.integration.resource; + +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; + +import java.util.Map; + +/** + * @author ldalves + */ +public interface CostReportConsumer { + + void consume(String csv); + + Map<Property, ResourceAllocation> fixedAllocations(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java index e30d2d55f77..bea34567752 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java @@ -41,5 +41,10 @@ public class ResourceAllocation { return new ResourceAllocation(cpuCores + allocation.cpuCores, memoryGb + allocation.memoryGb, diskGb + allocation.diskGb); } + /** Returns a copy of this with each resource multiplied by given factor */ + public ResourceAllocation multiply(double multiplicand) { + return new ResourceAllocation(cpuCores * multiplicand, memoryGb * multiplicand, diskGb * multiplicand); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 927a0ae2fe5..2d7b9675184 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -10,8 +10,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; -import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import java.time.Duration; import java.util.Collections; @@ -53,11 +51,13 @@ public class ControllerMaintenance extends AbstractComponent { private final RotationStatusUpdater rotationStatusUpdater; @SuppressWarnings("unused") // instantiated by Dependency Injection - public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, - JobControl jobControl, Metric metric, - CostReportConsumer reportConsumer, + public ControllerMaintenance(MaintainerConfig maintainerConfig, + ApiAuthorityConfig apiAuthorityConfig, + Controller controller, + CuratorDb curator, + JobControl jobControl, + Metric metric, Billing billing, - SelfHostedCostConfig selfHostedCostConfig, AwsEventFetcher awsEventFetcher) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); this.jobControl = jobControl; @@ -78,7 +78,7 @@ public class ControllerMaintenance extends AbstractComponent { osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl); - costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, selfHostedCostConfig); + costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), jobControl, controller.serviceRegistry().costReportConsumer()); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(30), jobControl, metric, controller.serviceRegistry().meteringService()); billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billing); awsEventReporterMaintainer = new AwsEventReporterMaintainer(controller, Duration.ofDays(1), jobControl, controller.serviceRegistry().issueHandler(), awsEventFetcher); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 20339b814a8..9c70f734baf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -1,14 +1,12 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.inject.Inject; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; +import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator; -import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; -import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import java.time.Clock; import java.time.Duration; @@ -25,24 +23,18 @@ public class CostReportMaintainer extends Maintainer { private final CostReportConsumer consumer; private final NodeRepository nodeRepository; private final Clock clock; - private final SelfHostedCostConfig selfHostedCostConfig; - @Inject - @SuppressWarnings("WeakerAccess") public CostReportMaintainer(Controller controller, Duration interval, - CostReportConsumer consumer, - JobControl jobControl, - SelfHostedCostConfig selfHostedCostConfig) { + JobControl jobControl, CostReportConsumer costReportConsumer) { super(controller, interval, jobControl, "CostReportMaintainer", EnumSet.of(SystemName.main)); - this.consumer = consumer; + this.consumer = costReportConsumer; this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); this.clock = controller.clock(); - this.selfHostedCostConfig = selfHostedCostConfig; } @Override protected void maintain() { - consumer.Consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, selfHostedCostConfig, CloudName.from("yahoo"))); + consumer.consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations(), CloudName.from("yahoo"))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java index ee633a19bfc..6ed9db7455c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java @@ -7,9 +7,9 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.restapi.Path; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; +import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.StringResponse; -import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import java.time.Clock; import java.util.Optional; @@ -23,13 +23,13 @@ public class CostApiHandler extends LoggingRequestHandler { private final Controller controller; private final NodeRepository nodeRepository; - private final SelfHostedCostConfig selfHostedCostConfig; + private final CostReportConsumer costReportConsumer; - public CostApiHandler(Context ctx, Controller controller, SelfHostedCostConfig selfHostedCostConfig) { + public CostApiHandler(Context ctx, Controller controller) { super(ctx); this.controller = controller; this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.selfHostedCostConfig = selfHostedCostConfig; + this.costReportConsumer = controller.serviceRegistry().costReportConsumer(); } @Override @@ -43,7 +43,7 @@ public class CostApiHandler extends LoggingRequestHandler { if (path.matches("/cost/v1/csv")) { Optional<String> cloudProperty = Optional.ofNullable(request.getProperty("cloud")); CloudName cloud = cloudProperty.map(CloudName::from).orElse(CloudName.defaultName()); - return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), selfHostedCostConfig, cloud)); + return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), costReportConsumer.fixedAllocations(), cloud)); } return ErrorResponse.notFoundError("Nothing at " + path); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java index 50c7732bf76..5cce865eb30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; -import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -32,7 +31,7 @@ public class CostCalculator { public static String resourceShareByPropertyToCsv(NodeRepository nodeRepository, Controller controller, Clock clock, - SelfHostedCostConfig selfHostedCostConfig, + Map<Property, ResourceAllocation> fixedAllocations, CloudName cloudName) { var date = new SimpleDateFormat("yyyy-MM-dd").format(Date.from(clock.instant())); @@ -62,14 +61,12 @@ public class CostCalculator { // Add fixed allocations from config if (cloudName.equals(CloudName.from("yahoo"))) { - for (var propertyEntry : selfHostedCostConfig.properties()) { - var property = new Property(propertyEntry.name()); + for (var kv : fixedAllocations.entrySet()) { + var property = kv.getKey(); var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO); - var fixedAllocation = new ResourceAllocation(propertyEntry.cpuCores() * SELF_HOSTED_DISCOUNT, - propertyEntry.memoryGb() * SELF_HOSTED_DISCOUNT, - propertyEntry.diskGb() * SELF_HOSTED_DISCOUNT); - allocationByProperty.put(property, allocation.plus(fixedAllocation)); - totalAllocation = totalAllocation.plus(fixedAllocation); + var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT); + allocationByProperty.put(property, allocation.plus(discountedFixedAllocation)); + totalAllocation = totalAllocation.plus(discountedFixedAllocation); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostReportConsumer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostReportConsumer.java deleted file mode 100644 index 51a664160ab..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostReportConsumer.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.yahoo.vespa.hosted.controller.restapi.cost; - -public interface CostReportConsumer { - void Consume(String csv); -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostReportConsumerMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostReportConsumerMock.java new file mode 100644 index 00000000000..5c16560b509 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostReportConsumerMock.java @@ -0,0 +1,38 @@ +package com.yahoo.vespa.hosted.controller.restapi.cost; + +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; + +import java.util.Map; +import java.util.function.Consumer; + +/** + * @author ldalves + */ +public class CostReportConsumerMock implements CostReportConsumer { + + private final Consumer<String> csvConsumer; + private final Map<Property, ResourceAllocation> fixedAllocations; + + public CostReportConsumerMock() { + this.csvConsumer = (ignored) -> {}; + this.fixedAllocations = Map.of(); + } + + public CostReportConsumerMock(Consumer<String> csvConsumer, Map<Property, ResourceAllocation> fixedAllocations) { + this.csvConsumer = csvConsumer; + this.fixedAllocations = Map.copyOf(fixedAllocations); + } + + @Override + public void consume(String csv) { + csvConsumer.accept(csv); + } + + @Override + public Map<Property, ResourceAllocation> fixedAllocations() { + return fixedAllocations; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/NoopCostReportConsumer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/NoopCostReportConsumer.java deleted file mode 100644 index f1406da1d4b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/NoopCostReportConsumer.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.yahoo.vespa.hosted.controller.restapi.cost; - -import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; - -public class NoopCostReportConsumer implements CostReportConsumer { - - @Inject - public NoopCostReportConsumer() {} - - @Override - public void Consume(String csv) { - // discard into the void - } -} diff --git a/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def b/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def deleted file mode 100644 index 059af1558a4..00000000000 --- a/controller-server/src/main/resources/configdefinitions/self-hosted-cost.def +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.hosted.controller.restapi.cost.config - -properties[].name string -properties[].cpuCores int -properties[].memoryGb int -properties[].diskGb int
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 0d15d863ab6..2b1cd2f17e8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; @@ -24,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIss import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; +import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumerMock; /** * A mock implementation of a {@link ServiceRegistry} for testing purposes. @@ -45,6 +47,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final DummyOwnershipIssues dummyOwnershipIssues = new DummyOwnershipIssues(); private final LoggingDeploymentIssues loggingDeploymentIssues = new LoggingDeploymentIssues(); private final MemoryEntityService memoryEntityService = new MemoryEntityService(); + private final CostReportConsumerMock costReportConsumerMock = new CostReportConsumerMock(); @Override public ConfigServer configServer() { @@ -102,6 +105,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override + public CostReportConsumer costReportConsumer() { + return costReportConsumerMock; + } + + @Override public NameService nameService() { return memoryNameService; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index 892da281667..bc68491d8dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -1,13 +1,15 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; -import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; +import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumerMock; import org.junit.Test; import java.time.Duration; import java.time.Instant; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -28,21 +30,15 @@ public class CostReportMaintainerTest { ZoneApiMock.newBuilder().withId("prod.eu-west-1").withCloud("yahoo").build()); addNodes(); - CostReportConsumer mockConsumer = csv -> assertEquals( - "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n" + + CostReportConsumerMock costReportConsumer = new CostReportConsumerMock( + (csv) -> assertEquals( + "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n" + "1970-01-01,Property1,96.0,96.0,2000.0,0.3055555555555555\n" + "1970-01-01,Property3,128.0,96.0,2000.0,0.3333333333333333\n" + "1970-01-01,Property2,160.0,96.0,2000.0,0.3611111111111111", - csv); - - SelfHostedCostConfig costConfig = new SelfHostedCostConfig.Builder() - .properties( - new SelfHostedCostConfig.Properties.Builder() - .name("Property3") - .cpuCores(256) - .memoryGb(192) - .diskGb(4000)) - .build(); + csv), + Map.of(new Property("Property3"), new ResourceAllocation(256, 192, 4000)) + ); tester.createTenant("tenant1", "app1", 1L); @@ -50,9 +46,8 @@ public class CostReportMaintainerTest { CostReportMaintainer maintainer = new CostReportMaintainer( tester.controller(), Duration.ofDays(1), - mockConsumer, new JobControl(tester.curator()), - costConfig); + costReportConsumer); maintainer.maintain(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index bc1c0c5013d..d1c7ac5d363 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -64,7 +64,6 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.restapi.cost.NoopCostReportConsumer'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher' />\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockBilling'/>\n" + |