diff options
Diffstat (limited to 'controller-api/src')
10 files changed, 161 insertions, 113 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ClusterId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ClusterId.java new file mode 100644 index 00000000000..0565a916dfa --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ClusterId.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import com.yahoo.config.provision.ClusterSpec; + +import java.util.Objects; + +/** + * DeploymentId x ClusterSpec.Id = ClusterId + * + * @author ogronnesby + */ +public class ClusterId { + private final DeploymentId deploymentId; + private final ClusterSpec.Id clusterId; + + public ClusterId(DeploymentId deploymentId, ClusterSpec.Id clusterId) { + this.deploymentId = deploymentId; + this.clusterId = clusterId; + } + + public DeploymentId deploymentId() { + return deploymentId; + } + + public ClusterSpec.Id clusterId() { + return clusterId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterId clusterId1 = (ClusterId) o; + return Objects.equals(deploymentId, clusterId1.deploymentId) && Objects.equals(clusterId, clusterId1.clusterId); + } + + @Override + public int hashCode() { + return Objects.hash(deploymentId, clusterId); + } + + @Override + public String toString() { + return "ClusterId{" + + "deploymentId=" + deploymentId + + ", clusterId=" + clusterId + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index be83fd8de48..bf16913d05a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; @@ -63,8 +62,6 @@ public interface ServiceRegistry { EndpointCertificateValidator endpointCertificateValidator(); - MeteringClient meteringService(); - ContactRetriever contactRetriever(); IssueHandler issueHandler(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java index 0efa225a437..7322c8e15f8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java @@ -65,12 +65,12 @@ public class ZtsClientMock implements ZtsClient { } @Override - public ZToken getRoleToken(AthenzDomain domain) { + public ZToken getRoleToken(AthenzDomain domain, Duration expiry) { throw new UnsupportedOperationException(); } @Override - public ZToken getRoleToken(AthenzRole athenzRole) { + public ZToken getRoleToken(AthenzRole athenzRole, Duration expiry) { throw new UnsupportedOperationException(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java index 6e60ec76199..b500cd1c133 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterSpec; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -131,6 +132,28 @@ public class Cluster { public Instant at() { return at; } public Optional<Instant> completion() { return completion; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ScalingEvent that = (ScalingEvent) o; + return Objects.equals(from, that.from) && Objects.equals(to, that.to) && Objects.equals(at, that.at) && Objects.equals(completion, that.completion); + } + + @Override + public int hashCode() { + return Objects.hash(from, to, at, completion); + } + + @Override + public String toString() { + return "ScalingEvent{" + + "from=" + from + + ", to=" + to + + ", at=" + at + + ", completion=" + completion + + '}'; + } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 77879699ab9..09ddcf4fa5e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import java.util.Comparator; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static ai.vespa.validation.Validation.require; @@ -18,7 +18,6 @@ import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; import static java.util.Comparator.naturalOrder; -import static java.util.stream.Collectors.toUnmodifiableList; /** * Specification for a deployment and/or test job to run: what zone, and whether it is a production test. @@ -27,6 +26,8 @@ import static java.util.stream.Collectors.toUnmodifiableList; */ public final class JobType implements Comparable<JobType> { + private static final RegionName unknown = RegionName.from("unknown"); + private final String jobName; private final ZoneId zone; private final boolean isProductionTest; @@ -38,18 +39,28 @@ public final class JobType implements Comparable<JobType> { } /** A system test in a test zone, or throws if no test zones are present.. */ - public static JobType systemTest(ZoneRegistry zones) { - return testIn(test, zones); + public static JobType systemTest(ZoneRegistry zones, CloudName cloud) { + return testIn(test, zones, cloud); } /** A staging test in a staging zone, or throws if no staging zones are present. */ - public static JobType stagingTest(ZoneRegistry zones){ - return testIn(staging, zones); + public static JobType stagingTest(ZoneRegistry zones, CloudName cloud){ + return testIn(staging, zones, cloud); } - private static JobType testIn(Environment environment, ZoneRegistry zones) { - return zones.zones().controllerUpgraded().in(environment).zones().stream().map(zone -> deploymentTo(zone.getId())) - .findFirst().orElseThrow(() -> new IllegalArgumentException("no zones in " + environment + " among " + zones.zones().controllerUpgraded().zones())); + /** Returns a test job in the given environment, preferring the given cloud, is possible; using the system cloud otherwise. */ + private static JobType testIn(Environment environment, ZoneRegistry zones, CloudName cloud) { + if (cloud == null) + return deploymentTo(ZoneId.from(environment, unknown)); + + ZoneList candidates = zones.zones().controllerUpgraded().in(environment); + if (candidates.in(cloud).zones().isEmpty()) + cloud = zones.systemZone().getCloudName(); + + return candidates.in(cloud).zones().stream().findFirst() + .map(zone -> deploymentTo(zone.getId())) + .orElseThrow(() -> new IllegalArgumentException("no zones in " + environment + " among " + + zones.zones().controllerUpgraded().zones())); } /** A deployment to the given dev region. */ @@ -118,27 +129,34 @@ public final class JobType implements Comparable<JobType> { throw new IllegalArgumentException("illegal serialized job type '" + raw + "'"); } - /** Creates a new job type from a job name, and a zone registry for looking up zones for the special system and staging test types. */ + /** + * Creates a new job type from a job name, and a zone registry for looking up zones for the special system and staging test types. + * Note: system and staging tests retrieved by job name always use the default cloud for the system! + */ public static JobType fromJobName(String jobName, ZoneRegistry zones) { - String[] parts = jobName.split("-", 2); - if (parts.length != 2) throw new IllegalArgumentException("job names must be 'system-test', 'staging-test', or environment and region parts, separated by '-', but got: " + jobName); - switch (parts[0]) { - case "system": return systemTest(zones); - case "staging": return stagingTest(zones); - case "production": return prod(parts[1]); - case "test": return test(parts[1]); - case "dev": return dev(parts[1]); - case "perf": return perf(parts[1]); - default: throw new IllegalArgumentException("job names must begin with one of: system, staging, production, test, dev, perf; but got: " + jobName); + switch (jobName) { + case "system-test": return systemTest(zones, null); + case "staging-test": return stagingTest(zones, null); } + String[] parts = jobName.split("-", 2); + if (parts.length == 2) + switch (parts[0]) { + case "production": return prod(parts[1]); + case "test": return test(parts[1]); + case "dev": return dev(parts[1]); + case "perf": return perf(parts[1]); + } + throw new IllegalArgumentException("job names must be 'system-test', 'staging-test', or <test|environment>-<region>, but got: " + jobName); } public static List<JobType> allIn(ZoneRegistry zones) { return zones.zones().reachable().zones().stream() .flatMap(zone -> zone.getEnvironment().isProduction() ? Stream.of(deploymentTo(zone.getId()), productionTestOf(zone.getId())) - : Stream.of(deploymentTo(zone.getId()))) + : zone.getEnvironment().isTest() ? Stream.of(deploymentTo(ZoneId.from(zone.getEnvironment(), unknown))) + : Stream.of(deploymentTo(zone.getId()))) + .distinct() .sorted(naturalOrder()) - .collect(toUnmodifiableList()); + .toList(); } /** A serialized form of this: {@code <environment>.<region>[.test]}; the inverse of {@link #ofSerialized(String)} */ @@ -152,6 +170,10 @@ public final class JobType implements Comparable<JobType> { /** Returns the zone for this job. */ public ZoneId zone() { + // sigh ... but the alternative is worse. + if (zone.region() == unknown) + throw new IllegalStateException("this job type was not initiated with a proper zone, programming error"); + return zone; } @@ -186,7 +208,7 @@ public final class JobType implements Comparable<JobType> { @Override public int compareTo(JobType other) { int result; - if (0 != (result = environment().compareTo(other.environment()))) return -result; + if (0 != (result = environment().compareTo(other.environment())) || environment().isTest()) return -result; if (0 != (result = zone.region().compareTo(other.zone.region()))) return result; return Boolean.compare(isProductionTest, other.isProductionTest); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java deleted file mode 100644 index 944a5eaf696..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.resource; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; - -import java.time.YearMonth; -import java.util.Collection; -import java.util.List; - -/** - * Consumes and retrieves snapshots of resources allocated per application. - * - * @author olaa - */ -public interface MeteringClient { - - void consume(Collection<ResourceSnapshot> resources); - - List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth); - - void refresh(); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java index 8ae12c0e7ac..49a24296fd3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java @@ -1,14 +1,21 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; -import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; + +import java.time.Instant; import java.time.LocalDate; import java.time.YearMonth; import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -22,12 +29,18 @@ public interface ResourceDatabaseClient { void refreshMaterializedView(); + void writeScalingEvents(ClusterId clusterId, Collection<Cluster.ScalingEvent> scalingEvents); + + Map<ClusterId, List<Cluster.ScalingEvent>> scalingEvents(Instant from, Instant to, DeploymentId deploymentId); + Set<YearMonth> getMonthsWithSnapshotsForTenant(TenantName tenantName); List<ResourceSnapshot> getRawSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth); Set<TenantName> getTenants(); + Instant getOldestSnapshotTimestamp(Set<DeploymentId> deployments); + default List<ResourceUsage> getResourceSnapshotsForMonth(TenantName tenantName, YearMonth month) { return getResourceSnapshotsForPeriod(tenantName, getMonthStartTimeStamp(month), getMonthEndTimeStamp(month)); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java index c680990e240..f3abfd37b38 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java @@ -1,17 +1,19 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; import java.math.BigDecimal; import java.time.Duration; import java.time.Instant; -import java.time.LocalDate; import java.time.YearMonth; -import java.time.ZoneId; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -33,6 +35,7 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { PlanRegistry planRegistry; Map<TenantName, Plan> planMap = new HashMap<>(); List<ResourceSnapshot> resourceSnapshots = new ArrayList<>(); + Map<ClusterId, List<Cluster.ScalingEvent>> scalingEvents = new HashMap<>(); private boolean hasRefreshedMaterializedView = false; public ResourceDatabaseClientMock(PlanRegistry planRegistry) { @@ -104,7 +107,7 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { public List<ResourceUsage> getResourceSnapshotsForPeriod(TenantName tenantName, long start, long end) { var tenantPlan = planMap.getOrDefault(tenantName, planRegistry.defaultPlan()); - var snapshotsPerDeployment = resourceSnapshots.stream() + return resourceSnapshots.stream() .filter(snapshot -> snapshot.getTimestamp().isAfter(Instant.ofEpochMilli(start))) .filter(snapshot -> snapshot.getTimestamp().isBefore(Instant.ofEpochMilli(end))) .filter(snapshot -> snapshot.getApplicationId().tenant().equals(tenantName)) @@ -117,8 +120,6 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); - - return snapshotsPerDeployment; } @Override @@ -126,6 +127,21 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { hasRefreshedMaterializedView = true; } + @Override + public Instant getOldestSnapshotTimestamp(Set<DeploymentId> deployments) { + return Instant.ofEpochMilli(987654L); + } + + @Override + public void writeScalingEvents(ClusterId clusterId, Collection<Cluster.ScalingEvent> scalingEvents) { + this.scalingEvents.put(clusterId, List.copyOf(scalingEvents)); + } + + @Override + public Map<ClusterId, List<Cluster.ScalingEvent>> scalingEvents(Instant from, Instant to, DeploymentId deploymentId) { + return Map.of(); + } + public void setPlan(TenantName tenant, Plan plan) { planMap.put(tenant, plan); } @@ -133,4 +149,9 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { public boolean hasRefreshedMaterializedView() { return hasRefreshedMaterializedView; } + + public List<ResourceSnapshot> resourceSnapshots() { + return resourceSnapshots; + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java deleted file mode 100644 index ca094f98607..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.stubs; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; - -import java.time.YearMonth; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * @author olaa - */ -public class MockMeteringClient implements MeteringClient { - - private Collection<ResourceSnapshot> resources = new ArrayList<>(); - private Optional<MeteringData> meteringData; - private boolean isRefreshed = false; - - @Override - public void consume(Collection<ResourceSnapshot> resources){ - this.resources = resources; - } - - @Override - public List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth) { - return new ArrayList<>(resources); - } - - @Override - public void refresh() { - isRefreshed = true; - } - - public Collection<ResourceSnapshot> consumedResources() { - return this.resources; - } - - public void setMeteringData(MeteringData meteringData) { - this.meteringData = Optional.of(meteringData); - this.resources = meteringData.getSnapshotHistory().entrySet().stream().map(Map.Entry::getValue).flatMap(List::stream).collect(Collectors.toList()); - } - - public boolean isRefreshed() { - return isRefreshed; - } -} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java index 6ff52bd5f03..bbe0c2bd458 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java @@ -48,4 +48,4 @@ public class JobTypeTest { assertTrue(JobType.test("snohetta").isProduction()); } -} +}
\ No newline at end of file |