diff options
7 files changed, 100 insertions, 32 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java new file mode 100644 index 00000000000..e6dec99b84c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.net.URI; +import java.util.Optional; + +/** + * Represents an operation to update or unset the archive URI value for a given tenant or cloud account. + * + * @author freva + */ +public class ArchiveUriUpdate { + private final Optional<TenantName> tenantName; + private final Optional<CloudAccount> cloudAccount; + private final Optional<URI> archiveUri; + + private ArchiveUriUpdate(Optional<TenantName> tenantName, Optional<CloudAccount> cloudAccount, Optional<URI> archiveUri) { + this.tenantName = tenantName; + this.cloudAccount = cloudAccount; + this.archiveUri = archiveUri; + } + + public Optional<TenantName> tenantName() { return tenantName; } + public Optional<CloudAccount> cloudAccount() { return cloudAccount; } + public Optional<URI> archiveUri() { return archiveUri; } + + public static ArchiveUriUpdate setArchiveUriFor(TenantName tenantName, URI archiveUri) { + return new ArchiveUriUpdate(Optional.of(tenantName), Optional.empty(), Optional.of(archiveUri)); + } + public static ArchiveUriUpdate deleteArchiveUriFor(TenantName tenantName) { + return new ArchiveUriUpdate(Optional.of(tenantName), Optional.empty(), Optional.empty()); + } + + public static ArchiveUriUpdate setArchiveUriFor(CloudAccount cloudAccount, URI archiveUri) { + return new ArchiveUriUpdate(Optional.empty(), Optional.of(cloudAccount), Optional.of(archiveUri)); + } + public static ArchiveUriUpdate deleteArchiveUriFor(CloudAccount cloudAccount) { + return new ArchiveUriUpdate(Optional.empty(), Optional.of(cloudAccount), Optional.empty()); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java new file mode 100644 index 00000000000..a0f6955b59f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.net.URI; +import java.util.Map; + +/** + * @author freva + */ +public record ArchiveUris(Map<TenantName, URI> tenantArchiveUris, Map<CloudAccount, URI> accountArchiveUris) { + public static final ArchiveUris EMPTY = new ArchiveUris(Map.of(), Map.of()); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 1768de8d012..4c5a67626ea 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -5,15 +5,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; -import javax.ws.rs.HeaderParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import java.net.URI; import java.util.List; import java.util.Map; @@ -48,14 +43,11 @@ public interface NodeRepository { /** Get node statistics such as cost and load from given zone */ NodeRepoStats getStats(ZoneId zone); - /** Get all archive URLs found in zone */ - Map<TenantName, URI> getArchiveUris(ZoneId zone); + /** Get all archive URIs found in zone */ + ArchiveUris getArchiveUris(ZoneId zone); - /** Update archive URL for given tenant */ - void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri); - - /** Remove archive URL for given tenant */ - void removeArchiveUri(ZoneId zone, TenantName tenantName); + /** Update some archive URI in the given zone */ + void updateArchiveUri(ZoneId zone, ArchiveUriUpdate archiveUriUpdate); /** Upgrade all nodes of given type to a new version */ void upgrade(ZoneId zone, NodeType type, Version version, boolean allowDowngrade); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java index 274d07bfc3b..172523eb261 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java @@ -16,6 +16,9 @@ public class ArchiveList { @JsonProperty("tenant") public String tenant; + @JsonProperty("account") + public String account; + @JsonProperty("uri") public String uri; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 0c8a50fa821..ddb1365f2de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -5,12 +5,13 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import com.yahoo.yolean.Exceptions; -import java.net.URI; import java.time.Duration; import java.util.HashMap; import java.util.HashSet; @@ -56,17 +57,19 @@ public class ArchiveUriUpdater extends ControllerMaintainer { int failures = 0; for (ZoneId zone : tenantsByZone.keySet()) { try { - Map<TenantName, URI> zoneArchiveUris = nodeRepository.getArchiveUris(zone); + ArchiveUris zoneArchiveUris = nodeRepository.getArchiveUris(zone); for (TenantName tenant : tenantsByZone.get(zone)) { archiveBucketDb.archiveUriFor(zone, tenant, true) - .filter(uri -> !uri.equals(zoneArchiveUris.get(tenant))) - .ifPresent(uri -> nodeRepository.setArchiveUri(zone, tenant, uri)); + .filter(uri -> !uri.equals(zoneArchiveUris.tenantArchiveUris().get(tenant))) + .ifPresent(uri -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.setArchiveUriFor(tenant, uri))); } - zoneArchiveUris.keySet().stream() + zoneArchiveUris.tenantArchiveUris().keySet().stream() .filter(tenant -> !tenantsByZone.get(zone).contains(tenant)) - .forEach(tenant -> nodeRepository.removeArchiveUri(zone, tenant)); + .forEach(tenant -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.deleteArchiveUriFor(tenant))); + + // TODO (freva): Update account archive URIs } catch (Exception e) { log.log(Level.WARNING, "Failed to update archive URI in " + zone + ". Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 37ef85b991b..297997365b0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; @@ -11,8 +12,10 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationStats; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Load; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; @@ -22,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.TargetVers import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; import java.net.URI; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -41,7 +45,7 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<ZoneId, TargetVersions> targetVersions = new ConcurrentHashMap<>(); private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new ConcurrentHashMap<>(); private final Map<DeploymentClusterId, BcpGroupInfo> bcpGroupInfos = new ConcurrentHashMap<>(); - private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new ConcurrentHashMap<>(); + private final Map<ZoneId, ArchiveUris> archiveUris = new ConcurrentHashMap<>(); private boolean allowPatching = true; private boolean hasSpareCapacity = false; @@ -118,18 +122,26 @@ public class NodeRepositoryMock implements NodeRepository { } @Override - public Map<TenantName, URI> getArchiveUris(ZoneId zone) { - return Map.copyOf(archiveUris.getOrDefault(zone, Map.of())); + public ArchiveUris getArchiveUris(ZoneId zone) { + return archiveUris.getOrDefault(zone, ArchiveUris.EMPTY); } @Override - public void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri) { - archiveUris.computeIfAbsent(zone, z -> new ConcurrentHashMap<>()).put(tenantName, archiveUri); - } - - @Override - public void removeArchiveUri(ZoneId zone, TenantName tenantName) { - Optional.ofNullable(archiveUris.get(zone)).ifPresent(map -> map.remove(tenantName)); + public void updateArchiveUri(ZoneId zone, ArchiveUriUpdate update) { + archiveUris.compute(zone, (z, prev) -> { + prev = prev == null ? ArchiveUris.EMPTY : prev; + if (update.tenantName().isPresent()) { + Map<TenantName, URI> updated = new HashMap<>(prev.tenantArchiveUris()); + update.archiveUri().ifPresentOrElse(uri -> updated.put(update.tenantName().get(), uri), + () -> updated.remove(update.tenantName().get())); + return new ArchiveUris(updated, prev.accountArchiveUris()); + } else { + Map<CloudAccount, URI> updated = new HashMap<>(prev.accountArchiveUris()); + update.archiveUri().ifPresentOrElse(uri -> updated.put(update.cloudAccount().get(), uri), + () -> updated.remove(update.cloudAccount().get())); + return new ArchiveUris(prev.tenantArchiveUris(), updated); + } + }); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index 8c44b39691c..de62a2fc48c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -62,7 +62,7 @@ public class ArchiveUriUpdaterTest { private void assertArchiveUris(Map<TenantName, String> expectedUris, ZoneId zone) { Map<TenantName, String> actualUris = tester.controller().serviceRegistry().configServer().nodeRepository() - .getArchiveUris(zone).entrySet().stream() + .getArchiveUris(zone).tenantArchiveUris().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString())); assertEquals(expectedUris, actualUris); } @@ -76,7 +76,7 @@ public class ArchiveUriUpdaterTest { private void setArchiveUriInNodeRepo(Map<TenantName, String> archiveUris, ZoneId zone) { NodeRepository nodeRepository = tester.controller().serviceRegistry().configServer().nodeRepository(); - archiveUris.forEach((tenant, uri) -> nodeRepository.setArchiveUri(zone, tenant, URI.create(uri))); + archiveUris.forEach((tenant, uri) -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.setArchiveUriFor(tenant, URI.create(uri)))); } private void deploy(DeploymentContext application, ZoneId zone) { |