From f4b248296b9322bbd59f8802ffe9f5d5b56c5ef0 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 27 Apr 2021 17:25:13 +0200 Subject: Expose content resource usage metrics from cluster-controller --- .../api/application/v4/model/ClusterMetrics.java | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'controller-api') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java index cb29d5854a0..0e11bcdccaf 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java @@ -15,7 +15,10 @@ public class ClusterMetrics { public static final String DOCUMENT_COUNT = "documentCount"; public static final String FEED_LATENCY = "feedLatency"; public static final String QUERY_LATENCY = "queryLatency"; - public static final String FEEDING_BLOCKED = "feedingBlocked"; + public static final String MEMORY_UTIL = "memoryUtil"; + public static final String MEMORY_FEED_BLOCK_LIMIT = "memoryFeedBlockLimit"; + public static final String DISK_UTIL = "diskUtil"; + public static final String DISK_FEED_BLOCK_LIMIT = "diskFeedBlockLimit"; private final String clusterId; private final String clusterType; @@ -55,8 +58,20 @@ public class ClusterMetrics { return Optional.ofNullable(metrics.get(QUERY_LATENCY)); } - public Optional feedingBlocked() { - return Optional.ofNullable(metrics.get(FEEDING_BLOCKED)); + public Optional memoryUtil() { + return Optional.ofNullable(metrics.get(MEMORY_UTIL)); + } + + public Optional memoryFeedBlockLimit() { + return Optional.ofNullable(metrics.get(MEMORY_FEED_BLOCK_LIMIT)); + } + + public Optional diskUtil() { + return Optional.ofNullable(metrics.get(DISK_UTIL)); + } + + public Optional diskFeedBlockLimit() { + return Optional.ofNullable(metrics.get(DISK_FEED_BLOCK_LIMIT)); } } -- cgit v1.2.3 From d6196844131e7339161b3db2bed8bdba287f2fbe Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 4 May 2021 10:55:16 +0200 Subject: Remove unnecessary interface and package exports --- .../api/integration/archive/ArchiveBucketDb.java | 16 ---------------- .../controller/api/integration/archive/package-info.java | 8 -------- .../controller/archive/CuratorArchiveBucketDb.java | 10 ++-------- .../vespa/hosted/controller/archive/package-info.java | 5 ----- .../controller/maintenance/ArchiveAccessMaintainer.java | 4 ++-- .../hosted/controller/maintenance/ArchiveUriUpdater.java | 4 ++-- 6 files changed, 6 insertions(+), 41 deletions(-) delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java (limited to 'controller-api') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java deleted file mode 100644 index 91c7cd68ea2..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.archive; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; - -import java.net.URI; -import java.util.Optional; -import java.util.Set; - -public interface ArchiveBucketDb { - - Optional archiveUriFor(ZoneId zoneId, TenantName tenant); - - Set buckets(ZoneId zoneId); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java deleted file mode 100644 index 067d05cd14e..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author freva - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.archive; - -import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 8bc2271825b..39d1fe1da57 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -1,7 +1,6 @@ // Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.archive; -import com.google.inject.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.FetchVector; @@ -9,10 +8,8 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import org.jetbrains.annotations.NotNull; import java.net.URI; import java.util.HashSet; @@ -27,7 +24,7 @@ import java.util.stream.Collectors; * * @author andreer */ -public class CuratorArchiveBucketDb implements ArchiveBucketDb { +public class CuratorArchiveBucketDb { /** * Due to policy limits, we can't put data for more than this many tenants in a bucket. @@ -47,14 +44,12 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { private final CuratorDb curatorDb; private final StringFlag bucketNameFlag; - @Inject public CuratorArchiveBucketDb(Controller controller) { this.archiveService = controller.serviceRegistry().archiveService(); this.curatorDb = controller.curator(); this.bucketNameFlag = Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.bindTo(controller.flagSource()); } - @Override public Optional archiveUriFor(ZoneId zoneId, TenantName tenant) { String bucketName = bucketNameFlag .with(FetchVector.Dimension.ZONE_ID, zoneId.value()) @@ -106,12 +101,10 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { } } - @Override public Set buckets(ZoneId zoneId) { return curatorDb.readArchiveBuckets(zoneId); } - @NotNull private Optional findAndUpdateArchiveUriCache(ZoneId zoneId, TenantName tenant, Set zoneBuckets) { Optional bucketName = zoneBuckets.stream() .filter(bucket -> bucket.tenants().contains(tenant)) @@ -132,4 +125,5 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); archiveUriCache.put(zoneId, bucketNameByTenant); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java deleted file mode 100644 index c93eb56d294..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.archive; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index 97385da468b..1a9889284e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Maps; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -23,7 +23,7 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { private static final String bucketCountMetricName = "archive.bucketCount"; - private final ArchiveBucketDb archiveBucketDb; + private final CuratorArchiveBucketDb archiveBucketDb; private final ArchiveService archiveService; private final ZoneRegistry zoneRegistry; private final Metric metric; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index faa4813e6b0..d2141b097b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -6,9 +6,9 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import java.net.URI; import java.time.Duration; @@ -28,7 +28,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; - private final ArchiveBucketDb archiveBucketDb; + private final CuratorArchiveBucketDb archiveBucketDb; public ArchiveUriUpdater(Controller controller, Duration duration) { super(controller, duration, ArchiveUriUpdater.class.getSimpleName(), SystemName.all()); -- cgit v1.2.3 From 467663800f2e56e4e9c7901330ef1a9d729acd37 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 4 May 2021 14:50:02 +0200 Subject: Revert "Remove unnecessary interface and package exports" --- .../api/integration/archive/ArchiveBucketDb.java | 16 ++++++++++++++++ .../controller/api/integration/archive/package-info.java | 8 ++++++++ .../com/yahoo/vespa/hosted/controller/Controller.java | 4 ++-- .../controller/archive/CuratorArchiveBucketDb.java | 10 ++++++++-- .../vespa/hosted/controller/archive/package-info.java | 5 +++++ .../controller/maintenance/ArchiveAccessMaintainer.java | 4 ++-- .../hosted/controller/maintenance/ArchiveUriUpdater.java | 4 ++-- 7 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java (limited to 'controller-api') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java new file mode 100644 index 00000000000..91c7cd68ea2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java @@ -0,0 +1,16 @@ +// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; + +import java.net.URI; +import java.util.Optional; +import java.util.Set; + +public interface ArchiveBucketDb { + + Optional archiveUriFor(ZoneId zoneId, TenantName tenant); + + Set buckets(ZoneId zoneId); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java new file mode 100644 index 00000000000..067d05cd14e --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author freva + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index e61a376730e..cd50382b1bf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -221,7 +221,8 @@ public class Controller extends AbstractComponent { if (version.isEmpty()) { throw new IllegalArgumentException("Invalid version '" + version.toFullString() + "'"); } - if (!clouds().contains(cloudName)) { + Set clouds = clouds(); + if (!clouds.contains(cloudName)) { throw new IllegalArgumentException("Cloud '" + cloudName + "' does not exist in this system"); } try (Lock lock = curator.lockOsVersions()) { @@ -305,5 +306,4 @@ public class Controller extends AbstractComponent { public NotificationsDb notificationsDb() { return notificationsDb; } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 39d1fe1da57..8bc2271825b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -1,6 +1,7 @@ // Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.archive; +import com.google.inject.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.FetchVector; @@ -8,8 +9,10 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import org.jetbrains.annotations.NotNull; import java.net.URI; import java.util.HashSet; @@ -24,7 +27,7 @@ import java.util.stream.Collectors; * * @author andreer */ -public class CuratorArchiveBucketDb { +public class CuratorArchiveBucketDb implements ArchiveBucketDb { /** * Due to policy limits, we can't put data for more than this many tenants in a bucket. @@ -44,12 +47,14 @@ public class CuratorArchiveBucketDb { private final CuratorDb curatorDb; private final StringFlag bucketNameFlag; + @Inject public CuratorArchiveBucketDb(Controller controller) { this.archiveService = controller.serviceRegistry().archiveService(); this.curatorDb = controller.curator(); this.bucketNameFlag = Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.bindTo(controller.flagSource()); } + @Override public Optional archiveUriFor(ZoneId zoneId, TenantName tenant) { String bucketName = bucketNameFlag .with(FetchVector.Dimension.ZONE_ID, zoneId.value()) @@ -101,10 +106,12 @@ public class CuratorArchiveBucketDb { } } + @Override public Set buckets(ZoneId zoneId) { return curatorDb.readArchiveBuckets(zoneId); } + @NotNull private Optional findAndUpdateArchiveUriCache(ZoneId zoneId, TenantName tenant, Set zoneBuckets) { Optional bucketName = zoneBuckets.stream() .filter(bucket -> bucket.tenants().contains(tenant)) @@ -125,5 +132,4 @@ public class CuratorArchiveBucketDb { .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); archiveUriCache.put(zoneId, bucketNameByTenant); } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java new file mode 100644 index 00000000000..c93eb56d294 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.archive; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index 1a9889284e1..97385da468b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Maps; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -23,7 +23,7 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { private static final String bucketCountMetricName = "archive.bucketCount"; - private final CuratorArchiveBucketDb archiveBucketDb; + private final ArchiveBucketDb archiveBucketDb; private final ArchiveService archiveService; private final ZoneRegistry zoneRegistry; private final Metric metric; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index d2141b097b3..faa4813e6b0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -6,9 +6,9 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import java.net.URI; import java.time.Duration; @@ -28,7 +28,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; - private final CuratorArchiveBucketDb archiveBucketDb; + private final ArchiveBucketDb archiveBucketDb; public ArchiveUriUpdater(Controller controller, Duration duration) { super(controller, duration, ArchiveUriUpdater.class.getSimpleName(), SystemName.all()); -- cgit v1.2.3 From 9934946722fbaa015c2d33a5a10c16733a231245 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 4 May 2021 10:55:16 +0200 Subject: Remove unnecessary interface and package export --- .../api/integration/archive/ArchiveBucketDb.java | 16 ---------------- .../controller/api/integration/archive/package-info.java | 2 +- .../controller/archive/CuratorArchiveBucketDb.java | 10 ++-------- .../vespa/hosted/controller/archive/package-info.java | 5 ----- .../controller/maintenance/ArchiveAccessMaintainer.java | 4 ++-- .../hosted/controller/maintenance/ArchiveUriUpdater.java | 4 ++-- 6 files changed, 7 insertions(+), 34 deletions(-) delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java (limited to 'controller-api') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java deleted file mode 100644 index 91c7cd68ea2..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.archive; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; - -import java.net.URI; -import java.util.Optional; -import java.util.Set; - -public interface ArchiveBucketDb { - - Optional archiveUriFor(ZoneId zoneId, TenantName tenant); - - Set buckets(ZoneId zoneId); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java index 067d05cd14e..2fa68f1f39d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/package-info.java @@ -5,4 +5,4 @@ @ExportPackage package com.yahoo.vespa.hosted.controller.api.integration.archive; -import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 8bc2271825b..39d1fe1da57 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -1,7 +1,6 @@ // Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.archive; -import com.google.inject.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.FetchVector; @@ -9,10 +8,8 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import org.jetbrains.annotations.NotNull; import java.net.URI; import java.util.HashSet; @@ -27,7 +24,7 @@ import java.util.stream.Collectors; * * @author andreer */ -public class CuratorArchiveBucketDb implements ArchiveBucketDb { +public class CuratorArchiveBucketDb { /** * Due to policy limits, we can't put data for more than this many tenants in a bucket. @@ -47,14 +44,12 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { private final CuratorDb curatorDb; private final StringFlag bucketNameFlag; - @Inject public CuratorArchiveBucketDb(Controller controller) { this.archiveService = controller.serviceRegistry().archiveService(); this.curatorDb = controller.curator(); this.bucketNameFlag = Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.bindTo(controller.flagSource()); } - @Override public Optional archiveUriFor(ZoneId zoneId, TenantName tenant) { String bucketName = bucketNameFlag .with(FetchVector.Dimension.ZONE_ID, zoneId.value()) @@ -106,12 +101,10 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { } } - @Override public Set buckets(ZoneId zoneId) { return curatorDb.readArchiveBuckets(zoneId); } - @NotNull private Optional findAndUpdateArchiveUriCache(ZoneId zoneId, TenantName tenant, Set zoneBuckets) { Optional bucketName = zoneBuckets.stream() .filter(bucket -> bucket.tenants().contains(tenant)) @@ -132,4 +125,5 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); archiveUriCache.put(zoneId, bucketNameByTenant); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java deleted file mode 100644 index c93eb56d294..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.archive; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index 97385da468b..1a9889284e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Maps; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -23,7 +23,7 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { private static final String bucketCountMetricName = "archive.bucketCount"; - private final ArchiveBucketDb archiveBucketDb; + private final CuratorArchiveBucketDb archiveBucketDb; private final ArchiveService archiveService; private final ZoneRegistry zoneRegistry; private final Metric metric; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index faa4813e6b0..d2141b097b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -6,9 +6,9 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import java.net.URI; import java.time.Duration; @@ -28,7 +28,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; - private final ArchiveBucketDb archiveBucketDb; + private final CuratorArchiveBucketDb archiveBucketDb; public ArchiveUriUpdater(Controller controller, Duration duration) { super(controller, duration, ArchiveUriUpdater.class.getSimpleName(), SystemName.all()); -- cgit v1.2.3 From 12375b621d185f5873ffa69e68a9d792ae9af6ad Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Tue, 4 May 2021 15:14:46 +0200 Subject: Maintain iam roles for tenants with prod deployment --- .../api/integration/aws/MockRoleService.java | 21 ++++++++ .../maintenance/TenantRoleMaintainer.java | 16 ++++-- .../integration/ServiceRegistryMock.java | 3 +- .../maintenance/TenantRoleMaintainerTest.java | 58 ++++++++++++++++++++++ 4 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockRoleService.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java (limited to 'controller-api') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockRoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockRoleService.java new file mode 100644 index 00000000000..f7c51f0bbc2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockRoleService.java @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.api.integration.aws; + +import com.yahoo.config.provision.TenantName; + +import java.util.List; + +public class MockRoleService extends NoopRoleService { + + private List maintainedTenants; + + @Override + public void maintainRoles(List tenants) { + maintainedTenants = List.copyOf(tenants); + } + + public List maintainedTenants() { + return maintainedTenants; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index e8b50a6b604..e3a7d91d7cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -2,22 +2,23 @@ package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class TenantRoleMaintainer extends ControllerMaintainer { - private final BooleanFlag provisionTenantRoles; - public TenantRoleMaintainer(Controller controller, Duration tenantRoleMaintainer) { super(controller, tenantRoleMaintainer); - provisionTenantRoles = Flags.PROVISION_TENANT_ROLES.bindTo(controller.flagSource()); } @Override @@ -26,10 +27,15 @@ public class TenantRoleMaintainer extends ControllerMaintainer { var tenants = controller().tenants().asList(); var tenantsWithRoles = tenants.stream() .map(Tenant::name) - // Only maintain a subset of the tenants - .filter(name -> provisionTenantRoles.with(FetchVector.Dimension.TENANT_ID, name.value()).value()) + .filter(this::hasProductionDeployment) .collect(Collectors.toList()); roleService.maintainRoles(tenantsWithRoles); return true; } + + private boolean hasProductionDeployment(TenantName tenant) { + return controller().applications().asList(tenant).stream() + .map(Application::productionInstances) + .noneMatch(Map::isEmpty); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 326928b9463..c11c6ba155a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -9,6 +9,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; @@ -66,7 +67,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ApplicationStoreMock applicationStoreMock = new ApplicationStoreMock(); private final MockRunDataStore mockRunDataStore = new MockRunDataStore(); private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); - private final RoleService roleService = new NoopRoleService(); + private final RoleService roleService = new MockRoleService(); private final BillingController billingController = new MockBillingController(); private final ContainerRegistryMock containerRegistry = new ContainerRegistryMock(); private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java new file mode 100644 index 00000000000..c023ec9b43d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java @@ -0,0 +1,58 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author mortent + */ +public class TenantRoleMaintainerTest { + + private final DeploymentTester tester = new DeploymentTester(); + + @Test + public void maintains_iam_roles_for_tenants_in_production() { + var devApp = tester.newDeploymentContext("tenant1", "app1", "default"); + var prodApp = tester.newDeploymentContext("tenant2", "app2", "default"); + ApplicationPackage appPackage = new ApplicationPackageBuilder() + .region("us-west-1") + .build(); + + // Deploy dev + devApp.runJob(JobType.devUsEast1, appPackage); + + // Deploy prod + prodApp.submit(appPackage).deploy(); + assertEquals(1, permanentDeployments(devApp.instance())); + assertEquals(1, permanentDeployments(prodApp.instance())); + + var maintainer = new TenantRoleMaintainer(tester.controller(), Duration.ofDays(1)); + maintainer.maintain(); + + var roleService = tester.controller().serviceRegistry().roleService(); + List tenantNames = ((MockRoleService) roleService).maintainedTenants(); + + assertEquals(1, tenantNames.size()); + assertEquals(prodApp.application().id().tenant(), tenantNames.get(0)); + } + + private long permanentDeployments(Instance instance) { + return tester.controller().applications().requireInstance(instance.id()).deployments().values().stream() + .filter(deployment -> !deployment.zone().environment().isTest()) + .count(); + } + +} -- cgit v1.2.3