diff options
13 files changed, 55 insertions, 40 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java index 05dea1f8567..8e7a261100a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsEventFetcher.java @@ -5,6 +5,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Issue; import java.util.List; public interface AwsEventFetcher { + List<CloudEvent> getEvents(String awsRegionName); Issue createIssue(CloudEvent event); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java index f716458542c..1e76917ebd5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java @@ -9,4 +9,5 @@ import com.yahoo.config.provision.ApplicationId; public interface Billing { void handleBilling(ApplicationId applicationId, String customerId); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java index 20b77703160..84156cd9d2a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java @@ -10,4 +10,5 @@ public class MockBilling implements Billing { @Override public void handleBilling(ApplicationId applicationId, String customerId) {} + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java index ab2a8425897..632dbaad419 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java @@ -1,16 +1,17 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; +import java.util.Collection; import java.util.List; /** - * Consumes and retrieves snapshots of resourses allocated per application. + * Consumes and retrieves snapshots of resources allocated per application. * * @author olaa */ public interface MeteringClient { - void consume(List<ResourceSnapshot> resources); + void consume(Collection<ResourceSnapshot> resources); MeteringInfo getResourceSnapshots(String tenantName, String applicationName); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringInfo.java index 8709315a83c..d6cb8f7fe76 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringInfo.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringInfo.java @@ -38,4 +38,5 @@ public class MeteringInfo { public Map<ApplicationId, List<ResourceSnapshot>> getSnapshotHistory() { return snapshotHistory; } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java index ed9c95bb795..10e1eb39c8a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapsh import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -16,11 +17,11 @@ import java.util.Optional; */ public class MockMeteringClient implements MeteringClient { - private List<ResourceSnapshot> resources = new ArrayList<>(); + private Collection<ResourceSnapshot> resources = new ArrayList<>(); private Optional<MeteringInfo> meteringInfo; @Override - public void consume(List<ResourceSnapshot> resources){ + public void consume(Collection<ResourceSnapshot> resources){ this.resources = resources; } @@ -32,7 +33,7 @@ public class MockMeteringClient implements MeteringClient { }); } - public List<ResourceSnapshot> consumedResources() { + public Collection<ResourceSnapshot> consumedResources() { return this.resources; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/AwsEventReporterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/AwsEventReporterMaintainer.java index 12bee2a7954..9a8adadd56e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/AwsEventReporterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/AwsEventReporterMaintainer.java @@ -16,10 +16,12 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** + * Automatically fetches scheduled events from AWS and submits issues detailing them. + * * @author mgimle - * Automatically fetches scheduled events from AWS and submits issues detailing them to Jira. */ public class AwsEventReporterMaintainer extends Maintainer { + private static final Logger log = Logger.getLogger(AwsEventReporterMaintainer.class.getName()); private final IssueHandler issueHandler; @@ -53,4 +55,5 @@ public class AwsEventReporterMaintainer extends Maintainer { } } } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java index c6956293adf..466a79b99dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java @@ -29,11 +29,12 @@ public class BillingMaintainer extends Maintainer { .map(tenant -> (CloudTenant) tenant) .forEach(cloudTenant -> controller().applications().asList(cloudTenant.name()) .stream() - .forEach( application -> { + .forEach(application -> { billing.handleBilling(application.id(), cloudTenant.billingInfo().customerId()); }) ); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 2fdee305f3d..a3221ed7366 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -77,7 +77,7 @@ public class ControllerMaintenance extends AbstractComponent { contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), jobControl, controller.serviceRegistry().costReportConsumer()); - resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(30), jobControl, metric, controller.serviceRegistry().meteringService()); + resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(1), jobControl, metric, controller.serviceRegistry().meteringService()); billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billing); awsEventReporterMaintainer = new AwsEventReporterMaintainer(controller, Duration.ofDays(1), jobControl, controller.serviceRegistry().issueHandler(), awsEventFetcher); rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval, jobControl); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 8dba0a3e813..c700ddac51c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapsh import java.time.Clock; import java.time.Duration; -import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -47,34 +47,33 @@ public class ResourceMeterMaintainer extends Maintainer { @Override protected void maintain() { - List<Node> nodes = getNodes(); - List<ResourceSnapshot> resourceSnapshots = getResourceSnapshots(nodes); - + Collection<ResourceSnapshot> resourceSnapshots = getResourceSnapshots(allocatedNodes()); meteringClient.consume(resourceSnapshots); metric.set(METERING_LAST_REPORTED, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); - metric.set(METERING_TOTAL_REPORTED, resourceSnapshots.stream() - .mapToDouble(r -> r.getCpuCores() + r.getMemoryGb() + r.getDiskGb()) // total metered resource usage, for alerting on drastic changes - .sum() - , metric.createContext(Collections.emptyMap())); + // total metered resource usage, for alerting on drastic changes + metric.set(METERING_TOTAL_REPORTED, + resourceSnapshots.stream().mapToDouble(r -> r.getCpuCores() + r.getMemoryGb() + r.getDiskGb()).sum(), + metric.createContext(Collections.emptyMap())); } - private List<Node> getNodes() { + private List<Node> allocatedNodes() { return controller().zoneRegistry().zones() .ofCloud(CloudName.from("aws")) .reachable().zones().stream() .flatMap(zone -> nodeRepository.list(zone.getId()).stream()) - .filter(node -> node.owner().isPresent() && !node.owner().get().tenant().value().equals("hosted-vespa")) - .filter(node -> node.state() == Node.State.active) + .filter(node -> node.owner().isPresent()) + .filter(node -> ! node.owner().get().tenant().value().equals("hosted-vespa")) .collect(Collectors.toList()); } - private List<ResourceSnapshot> getResourceSnapshots(List<Node> nodes) { - return new ArrayList<>(nodes.stream() - .filter(node -> node.owner().isPresent()) - .collect(Collectors.groupingBy(node -> node.owner().get(), - Collectors.collectingAndThen(Collectors.toList(), nodeList -> ResourceSnapshot.from(nodeList, clock.instant())) - )).values()); + private Collection<ResourceSnapshot> getResourceSnapshots(List<Node> nodes) { + return nodes.stream() + .collect(Collectors.groupingBy(node -> node.owner().get(), + Collectors.collectingAndThen(Collectors.toList(), + nodeList -> ResourceSnapshot.from(nodeList, + clock.instant())) + )).values(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index d8d7995f382..f28ce83e643 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; import java.time.Duration; +import java.util.Collection; import java.util.List; import static org.junit.Assert.assertEquals; @@ -36,7 +37,7 @@ public class ResourceMeterMaintainerTest { ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), metrics, snapshotConsumer); resourceMeterMaintainer.maintain(); - List<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); + Collection<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); // The mocked repository contains two applications, so we should also consume two ResourceSnapshots assertEquals(2, consumedResources.size()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index 48f846d5e7f..03cd3dd4019 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -11,6 +11,7 @@ import java.util.function.Function; import java.util.stream.Collectors; public class CapacityChecker { + private List<Node> hosts; Map<String, Node> nodeMap; private Map<Node, List<Node>> nodeChildren; @@ -523,4 +524,5 @@ public class CapacityChecker { return out.toString(); } } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java index 3c47e418b94..6a6fbf3aed0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java @@ -21,6 +21,7 @@ import java.util.*; * @author mgimle */ public class CapacityReportMaintainer extends Maintainer { + private final Metric metric; private final NodeRepository nodeRepository; private static final Logger log = Logger.getLogger(CapacityReportMaintainer.class.getName()); @@ -35,20 +36,21 @@ public class CapacityReportMaintainer extends Maintainer { @Override protected void maintain() { - if (!nodeRepository.zone().cloud().value().equals("aws")) { - CapacityChecker capacityChecker = new CapacityChecker(this.nodeRepository); - List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts(); - if (overcommittedHosts.size() != 0) { - log.log(LogLevel.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedHosts.size(), - overcommittedHosts.stream().map(Node::hostname).collect(Collectors.joining(", ")))); - } - metric.set("overcommittedHosts", overcommittedHosts.size(), null); - - Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); - if (failurePath.isPresent()) { - int worstCaseHostLoss = failurePath.get().hostsCausingFailure.size(); - metric.set("spareHostCapacity", worstCaseHostLoss - 1, null); - } + if (nodeRepository.zone().cloud().value().equals("aws")) return; // Hosts and nodes are 1-1 + + CapacityChecker capacityChecker = new CapacityChecker(this.nodeRepository); + List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts(); + if (overcommittedHosts.size() != 0) { + log.log(LogLevel.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedHosts.size(), + overcommittedHosts.stream().map(Node::hostname).collect(Collectors.joining(", ")))); + } + metric.set("overcommittedHosts", overcommittedHosts.size(), null); + + Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); + if (failurePath.isPresent()) { + int worstCaseHostLoss = failurePath.get().hostsCausingFailure.size(); + metric.set("spareHostCapacity", worstCaseHostLoss - 1, null); } } + } |