diff options
author | HÃ¥kon Hallingstad <hakon.hallingstad@gmail.com> | 2023-02-27 09:21:27 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-27 09:21:27 +0100 |
commit | 538d6baedd5f716e83e7a7f2ada96edfdc3fa08c (patch) | |
tree | 89c90a21a4cdd99400da07a86b2ebd2a9d4806a7 | |
parent | 04e33a7499b3d263ed73fcc6569a7c011402aafe (diff) | |
parent | 60b28199f48d150fb0734cde4a20fd2f7a7fab0b (diff) |
Merge pull request #26178 from vespa-engine/freva/archive-uris
Allow patching account archive URIs in node-repo
22 files changed, 465 insertions, 287 deletions
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 8d19925a886..00a805790f1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.WireguardKey; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; @@ -67,7 +68,7 @@ public class RealNodeRepositoryTest { for (int i = 0; i < 3; i++) { try { int port = findRandomOpenPort(); - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, CloudAccount.empty), Networking.enable); + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, SystemName.main, CloudAccount.empty), Networking.enable); ConfigServerApi configServerApi = ConfigServerApiImpl.createForTesting( List.of(URI.create("http://127.0.0.1:" + port))); waitForJdiscContainerToServe(configServerApi); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index fd75644d914..510c4041efb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -28,7 +28,7 @@ import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.JobControlFlags; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; -import com.yahoo.vespa.hosted.provision.provisioning.ArchiveUris; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUriManager; import com.yahoo.vespa.hosted.provision.provisioning.ContainerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; @@ -57,7 +57,7 @@ public class NodeRepository extends AbstractComponent { private final InfrastructureVersions infrastructureVersions; private final FirmwareChecks firmwareChecks; private final ContainerImages containerImages; - private final ArchiveUris archiveUris; + private final ArchiveUriManager archiveUriManager; private final JobControl jobControl; private final Applications applications; private final LoadBalancers loadBalancers; @@ -134,7 +134,7 @@ public class NodeRepository extends AbstractComponent { this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); this.containerImages = new ContainerImages(containerImage, tenantContainerImage, tenantGpuContainerImage); - this.archiveUris = new ArchiveUris(db, zone); + this.archiveUriManager = new ArchiveUriManager(db, zone); this.jobControl = new JobControl(new JobControlFlags(db, flagSource)); this.loadBalancers = new LoadBalancers(db); this.metricsDb = metricsDb; @@ -166,7 +166,7 @@ public class NodeRepository extends AbstractComponent { public ContainerImages containerImages() { return containerImages; } /** Returns the archive URIs to use for nodes in this. */ - public ArchiveUris archiveUris() { return archiveUris; } + public ArchiveUriManager archiveUriManager() { return archiveUriManager; } /** Returns the status of maintenance jobs managed by this. */ public JobControl jobControl() { return jobControl; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java new file mode 100644 index 00000000000..27488e4027c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.lang.CachedSupplier; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; + +import java.time.Duration; +import java.util.Optional; +import java.util.function.Function; +import java.util.logging.Logger; + +/** + * Thread safe class to get and set archive URI for given account and tenants. + * + * @author freva + */ +public class ArchiveUriManager { + + private static final Logger log = Logger.getLogger(ArchiveUriManager.class.getName()); + private static final Duration cacheTtl = Duration.ofMinutes(1); + + private final CuratorDb db; + private final Zone zone; + private final CachedSupplier<ArchiveUris> archiveUris; + + public ArchiveUriManager(CuratorDb db, Zone zone) { + this.db = db; + this.zone = zone; + this.archiveUris = new CachedSupplier<>(db::readArchiveUris, cacheTtl); + } + + public ArchiveUris archiveUris() { + return archiveUris.get(); + } + + /** Returns the archive URI to use for given node */ + public Optional<String> archiveUriFor(Node node) { + if (node.allocation().isEmpty()) return Optional.empty(); + ApplicationId app = node.allocation().get().owner(); + + return Optional.ofNullable(node.cloudAccount().isEnclave(zone) ? + archiveUris.get().accountArchiveUris().get(node.cloudAccount()) : + archiveUris.get().tenantArchiveUris().get(app.tenant())) + .map(uri -> { + StringBuilder sb = new StringBuilder(100).append(uri) + .append(app.application().value()).append('/') + .append(app.instance().value()).append('/') + .append(node.allocation().get().membership().cluster().id().value()).append('/'); + + for (char c: node.hostname().toCharArray()) { + if (c == '.') break; + sb.append(c); + } + + return sb.append('/').toString(); + }); + } + + /** Set (or remove, if archiveURI is empty) archive URI to use for given tenant */ + public void setArchiveUri(TenantName tenant, Optional<String> archiveUri) { + setArchiveUri(au -> au.with(tenant, archiveUri)); + } + + /** Set (or remove, if archiveURI is empty) archive URI to use for given account */ + public void setArchiveUri(CloudAccount account, Optional<String> archiveUri) { + if (!account.isEnclave(zone) || account.isUnspecified()) + throw new IllegalArgumentException("Cannot set archive URI for non-enclave account: " + account); + setArchiveUri(au -> au.with(account, archiveUri)); + } + + private void setArchiveUri(Function<ArchiveUris, ArchiveUris> mapper) { + try (Lock lock = db.lockArchiveUris()) { + ArchiveUris archiveUris = db.readArchiveUris(); + ArchiveUris updated = mapper.apply(archiveUris); + if (archiveUris.equals(updated)) return; // No change + + db.writeArchiveUris(updated); + this.archiveUris.invalidate(); // Throw away current cache + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java new file mode 100644 index 00000000000..7ec9bd36744 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +public record ArchiveUris(Map<TenantName, String> tenantArchiveUris, Map<CloudAccount, String> accountArchiveUris) { + private static final Pattern validUriPattern = Pattern.compile("[a-z0-9]+://(?:(?:[a-z0-9]+(?:[-_][a-z0-9.]+)*)+/)+"); + + public ArchiveUris(Map<TenantName, String> tenantArchiveUris, Map<CloudAccount, String> accountArchiveUris) { + this.tenantArchiveUris = Map.copyOf(tenantArchiveUris); + this.accountArchiveUris = Map.copyOf(accountArchiveUris); + } + + public ArchiveUris with(TenantName tenant, Optional<String> archiveUri) { + Map<TenantName, String> updated = updateIfChanged(tenantArchiveUris, tenant, archiveUri); + if (updated == tenantArchiveUris) return this; + return new ArchiveUris(updated, accountArchiveUris); + } + + public ArchiveUris with(CloudAccount account, Optional<String> archiveUri) { + Map<CloudAccount, String> updated = updateIfChanged(accountArchiveUris, account, archiveUri); + if (updated == accountArchiveUris) return this; + return new ArchiveUris(tenantArchiveUris, updated); + } + + private static <T> Map<T, String> updateIfChanged(Map<T, String> current, T key, Optional<String> archiveUri) { + archiveUri = archiveUri.map(ArchiveUris::normalizeUri); + if (Optional.ofNullable(current.get(key)).equals(archiveUri)) return current; + Map<T, String> updated = new HashMap<>(current); + archiveUri.ifPresentOrElse(uri -> updated.put(key, uri), () -> updated.remove(key)); + return updated; + } + + static String normalizeUri(String uri) { + if (!uri.endsWith("/")) uri = uri + "/"; + if (!validUriPattern.matcher(uri).matches()) + throw new IllegalArgumentException("Invalid archive URI: " + uri); + return uri; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java new file mode 100644 index 00000000000..3bf37816dd4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Serializer for archive URIs that are set per tenant and per account. + * + * @author freva + */ +public class ArchiveUriSerializer { + + private ArchiveUriSerializer() {} + + public static byte[] toJson(ArchiveUris archiveUris) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + Cursor tenantObject = root.setObject("tenant"); + archiveUris.tenantArchiveUris().forEach((tenant, uri) -> tenantObject.setString(tenant.value(), uri)); + + Cursor accountObject = root.setObject("account"); + archiveUris.accountArchiveUris().forEach((account, uri) -> accountObject.setString(account.value(), uri)); + + try { + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static ArchiveUris fromJson(byte[] data) { + Inspector inspector = SlimeUtils.jsonToSlime(data).get(); + + Map<TenantName, String> tenantArchiveUris = new HashMap<>(); + inspector.field("tenant").traverse((ObjectTraverser) (key, value) -> tenantArchiveUris.put(TenantName.from(key), value.asString())); + + Map<CloudAccount, String> accountArchiveUris = new HashMap<>(); + inspector.field("account").traverse((ObjectTraverser) (key, value) -> accountArchiveUris.put(CloudAccount.from(key), value.asString())); + + return new ArchiveUris(tenantArchiveUris, accountArchiveUris); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index c25dfe9f1e2..5aea6858f60 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -21,6 +20,7 @@ import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -70,7 +70,7 @@ public class CuratorDb { private static final Path infrastructureVersionsPath = root.append("infrastructureVersions"); private static final Path osVersionsPath = root.append("osVersions"); private static final Path firmwareCheckPath = root.append("firmwareCheck"); - private static final Path archiveUrisPath = root.append("archiveUris"); + private static final Path archiveUrisPath = root.append("archiveUri"); private static final Duration defaultLockTimeout = Duration.ofMinutes(1); @@ -102,6 +102,7 @@ public class CuratorDb { db.create(archiveUrisPath); db.create(loadBalancersPath); provisionIndexCounter.initialize(100); + CuratorOperations.delete(root.append("archiveUris").toString()); // TODO (freva): March 2023 } /** Adds a set of nodes. Rollbacks/fails transaction if any node is not in the expected state. */ @@ -375,16 +376,16 @@ public class CuratorDb { // Archive URIs ----------------------------------------------------------- - public void writeArchiveUris(Map<TenantName, String> archiveUris) { - byte[] data = TenantArchiveUriSerializer.toJson(archiveUris); + public void writeArchiveUris(ArchiveUris archiveUris) { + byte[] data = ArchiveUriSerializer.toJson(archiveUris); NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(archiveUrisPath.getAbsolute(), data)); transaction.commit(); } - public Map<TenantName, String> readArchiveUris() { - return read(archiveUrisPath, TenantArchiveUriSerializer::fromJson).orElseGet(Map::of); + public ArchiveUris readArchiveUris() { + return read(archiveUrisPath, ArchiveUriSerializer::fromJson).orElseGet(() -> new ArchiveUris(Map.of(), Map.of())); } public Lock lockArchiveUris() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java deleted file mode 100644 index d381d25704a..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Map; -import java.util.TreeMap; - -/** - * Serializer for archive URIs that are set per tenant. - * - * @author freva - */ -public class TenantArchiveUriSerializer { - - private TenantArchiveUriSerializer() {} - - public static byte[] toJson(Map<TenantName, String> archiveUrisByTenantName) { - Slime slime = new Slime(); - Cursor object = slime.setObject(); - archiveUrisByTenantName.forEach((tenantName, archiveUri) -> - object.setString(tenantName.value(), archiveUri)); - try { - return SlimeUtils.toJsonBytes(slime); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static Map<TenantName, String> fromJson(byte[] data) { - Map<TenantName, String> archiveUrisByTenantName = new TreeMap<>(); // Use TreeMap to sort by tenant name - Inspector inspector = SlimeUtils.jsonToSlime(data).get(); - inspector.traverse((ObjectTraverser) (key, value) -> - archiveUrisByTenantName.put(TenantName.from(key), value.asString())); - return archiveUrisByTenantName; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java deleted file mode 100644 index 285365a9c6d..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; -import com.yahoo.lang.CachedSupplier; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; - -import java.time.Duration; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; - -/** - * Thread safe class to get and set archive URI for given tenants. Archive URIs are stored in ZooKeeper so that - * nodes within the same tenant have the same archive URI from all the config servers. - * - * @author freva - */ -public class ArchiveUris { - - private static final Logger log = Logger.getLogger(ArchiveUris.class.getName()); - private static final Pattern validUriPattern = Pattern.compile("[a-z0-9]+://(?:(?:[a-z0-9]+(?:[-_][a-z0-9.]+)*)+/)+"); - private static final Duration cacheTtl = Duration.ofMinutes(1); - - private final CuratorDb db; - private final CachedSupplier<Map<TenantName, String>> archiveUris; - private final Zone zone; - - public ArchiveUris(CuratorDb db, Zone zone) { - this.db = db; - this.archiveUris = new CachedSupplier<>(db::readArchiveUris, cacheTtl); - this.zone = zone; - } - - /** Returns the current archive URI for each tenant */ - public Map<TenantName, String> getArchiveUris() { - return archiveUris.get(); - } - - /** Returns the archive URI to use for given tenant */ - private Optional<String> archiveUriFor(TenantName tenant) { - return Optional.ofNullable(archiveUris.get().get(tenant)); - } - - /** Returns the archive URI to use for given node */ - public Optional<String> archiveUriFor(Node node) { - if (node.cloudAccount().isEnclave(zone)) return Optional.empty(); // TODO (freva): Implement for exclave - - return node.allocation().map(Allocation::owner) - .flatMap(app -> archiveUriFor(app.tenant()) - .map(uri -> { - StringBuilder sb = new StringBuilder(100).append(uri) - .append(app.application().value()).append('/') - .append(app.instance().value()).append('/') - .append(node.allocation().get().membership().cluster().id().value()).append('/'); - - for (char c: node.hostname().toCharArray()) { - if (c == '.') break; - sb.append(c); - } - - return sb.append('/').toString(); - })); - } - - /** Set (or remove, if archiveURI is empty) archive URI to use for given tenant */ - public void setArchiveUri(TenantName tenant, Optional<String> archiveUri) { - try (Lock lock = db.lockArchiveUris()) { - Map<TenantName, String> archiveUris = new TreeMap<>(db.readArchiveUris()); - if (Optional.ofNullable(archiveUris.get(tenant)).equals(archiveUri)) return; // No change - - archiveUri.map(ArchiveUris::normalizeUri).ifPresentOrElse(uri -> archiveUris.put(tenant, uri), - () -> archiveUris.remove(tenant)); - db.writeArchiveUris(archiveUris); - this.archiveUris.invalidate(); // Throw away current cache - log.log(Level.FINE, () -> archiveUri.map(s -> "Set archive URI for " + tenant + " to " + s) - .orElseGet(() -> "Remove archive URI for " + tenant)); - } - } - - static String normalizeUri(String uri) { - if (!uri.endsWith("/")) uri = uri + "/"; - if (!validUriPattern.matcher(uri).matches()) - throw new IllegalArgumentException("Invalid archive URI: " + uri); - return uri; - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java index 3cff3c5e05f..84c82d314c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; + +import java.util.Map; /** * Returns tenant archive URIs. @@ -13,11 +16,22 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; public class ArchiveResponse extends SlimeJsonResponse { public ArchiveResponse(NodeRepository nodeRepository) { + ArchiveUris archiveUris = nodeRepository.archiveUriManager().archiveUris(); Cursor archivesArray = slime.setObject().setArray("archives"); - nodeRepository.archiveUris().getArchiveUris().forEach((tenant, uri) -> { + + archiveUris.tenantArchiveUris().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> { + Cursor archiveObject = archivesArray.addObject(); + archiveObject.setString("tenant", entry.getKey().value()); + archiveObject.setString("uri", entry.getValue()); + }); + archiveUris.accountArchiveUris().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> { Cursor archiveObject = archivesArray.addObject(); - archiveObject.setString("tenant", tenant.value()); - archiveObject.setString("uri", uri); + archiveObject.setString("account", entry.getKey().value()); + archiveObject.setString("uri", entry.getValue()); }); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 49e6265b6da..6bf07077c81 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -185,7 +185,7 @@ class NodesResponse extends SlimeJsonResponse { node.reports().toSlime(object, "reports"); node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); - nodeRepository.archiveUris().archiveUriFor(node).ifPresent(uri -> object.setString("archiveUri", uri)); + nodeRepository.archiveUriManager().archiveUriFor(node).ifPresent(uri -> object.setString("archiveUri", uri)); trustedCertsToSlime(node.trustedCertificates(), object); if (!node.cloudAccount().isUnspecified()) { object.setString("cloudAccount", node.cloudAccount().value()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 543901621d0..0fbe912812e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -186,9 +186,9 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new MessageResponse("Updated " + patcher.application()); } } - else if (path.matches("/nodes/v2/archive/{tenant}")) { + else if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}") || path.matches("/nodes/v2/archive/{key}") /* TODO (freva): Remove March 2023 */) { String uri = requiredField(toSlime(request), "uri", Inspector::asString); - return setTenantArchiveUri(path.get("tenant"), Optional.of(uri)); + return setArchiveUri(path.get("key"), Optional.of(uri), !path.getPath().segments().get(3).equals("account")); } else if (path.matches("/nodes/v2/upgrade/{nodeType}")) { return setTargetVersions(path.get("nodeType"), toSlime(request)); @@ -229,7 +229,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { private HttpResponse handleDELETE(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/nodes/v2/node/{hostname}")) return deleteNode(path.get("hostname")); - if (path.matches("/nodes/v2/archive/{tenant}")) return setTenantArchiveUri(path.get("tenant"), Optional.empty()); + if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}") || path.matches("/nodes/v2/archive/{key}") /* TODO (freva): Remove March 2023) */) + return setArchiveUri(path.get("key"), Optional.empty(), !path.getPath().segments().get(3).equals("account")); if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); @@ -422,9 +423,10 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new MessageResponse("Will request firmware checks on all hosts."); } - private HttpResponse setTenantArchiveUri(String tenant, Optional<String> archiveUri) { - nodeRepository.archiveUris().setArchiveUri(TenantName.from(tenant), archiveUri); - return new MessageResponse(archiveUri.map(a -> "Updated").orElse("Removed") + " archive URI for " + tenant); + private HttpResponse setArchiveUri(String key, Optional<String> archiveUri, boolean isTenant) { + if (isTenant) nodeRepository.archiveUriManager().setArchiveUri(TenantName.from(key), archiveUri); + else nodeRepository.archiveUriManager().setArchiveUri(CloudAccount.from(key), archiveUri); + return new MessageResponse(archiveUri.map(a -> "Updated").orElse("Removed") + " archive URI for " + key); } private static String hostnamesAsString(List<Node> nodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index f0f85b6523f..4f88a10dff0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; /** * For running NodeRepository API with some mocked data. @@ -11,12 +12,15 @@ import com.yahoo.config.provision.CloudAccount; */ public class ContainerConfig { - public static String servicesXmlV2(int port, CloudAccount cloudAccount) { + public static String servicesXmlV2(int port, SystemName systemName, CloudAccount cloudAccount) { return """ <container version='1.0'> <config name="container.handler.threadpool"> <maxthreads>20</maxthreads> </config> + <config name="cloud.config.configserver"> + <system>%s</system> + </config> <config name="config.provisioning.cloud"> <account>%s</account> </config> @@ -47,7 +51,7 @@ public class ContainerConfig { <server id='myServer' port='%s'/> </http> </container> - """.formatted(cloudAccount.value(), port); + """.formatted(systemName.value(), cloudAccount.value(), port); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java new file mode 100644 index 00000000000..8ee72d12f57 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; + +/** + * @author freva + */ +public class ArchiveUriManagerTest { + + @Test + public void archive_uri() { + ApplicationId app1 = ApplicationId.from("vespa", "music", "main"); + ApplicationId app2 = ApplicationId.from("yahoo", "music", "main"); + CloudAccount account1 = CloudAccount.from("123456789012"); + CloudAccount account2 = CloudAccount.from("210987654321"); + CloudAccount accountSystem = CloudAccount.from("555444333222"); + ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder() + .zone(new Zone(Cloud.builder().account(accountSystem).build(), SystemName.Public, Environment.prod, RegionName.defaultName())) + .build().nodeRepository().archiveUriManager(); + + // Initially no uris are set + assertFalse(archiveUriManager.archiveUriFor(createNode(null, null)).isPresent()); + assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account1)).isPresent()); + + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/dir")); + archiveUriManager.setArchiveUri(account1, Optional.of("scheme://account-bucket/dir")); + assertThrows(IllegalArgumentException.class, () -> archiveUriManager.setArchiveUri(accountSystem, Optional.of("scheme://something"))); + assertThrows(IllegalArgumentException.class, () -> archiveUriManager.setArchiveUri(CloudAccount.empty, Optional.of("scheme://something"))); + + assertFalse(archiveUriManager.archiveUriFor(createNode(null, null)).isPresent()); // Not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(null, account1)).isPresent()); // URI set for this account, but not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(null, account2)).isPresent()); // Not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(app2, null)).isPresent()); // No URI set for this tenant or account + assertEquals("scheme://tenant-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + assertEquals("scheme://account-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence + assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account2)).isPresent()); // URI set for this tenant, but is ignored because enclave account + assertEquals("scheme://tenant-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt + } + + private Node createNode(ApplicationId appId, CloudAccount account) { + Node.Builder nodeBuilder = Node.create("id", "h432a.prod.us-south-1.vespa.domain.tld", new Flavor(NodeResources.unspecified()), Node.State.parked, NodeType.tenant); + Optional.ofNullable(appId) + .map(app -> new Allocation(app, + ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), + NodeResources.unspecified(), + Generation.initial(), + false)) + .ifPresent(nodeBuilder::allocation); + Optional.ofNullable(account).ifPresent(nodeBuilder::cloudAccount); + return nodeBuilder.build(); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java new file mode 100644 index 00000000000..9770d60b27a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.hosted.provision.archive; + + +import com.yahoo.config.provision.TenantName; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.provision.archive.ArchiveUris.normalizeUri; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ArchiveUrisTest { + + @Test + void normalize_test() { + assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123")); + assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123/")); + assertEquals("s3://my-bucket-prod.region/my-tenant-123/", normalizeUri("s3://my-bucket-prod.region/my-tenant-123/")); + assertEquals("s3://my-bucket-prod.region/my-tenant_123/", normalizeUri("s3://my-bucket-prod.region/my-tenant_123/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("domain/dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain/dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain//dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal:dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/-illegal-dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/_illegal-dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir-/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir_/")); + } + + @Test + void updates_in_place_if_possible() { + TenantName t1 = TenantName.from("t1"); + + ArchiveUris uris0 = new ArchiveUris(Map.of(), Map.of()); + ArchiveUris uris1 = uris0.with(t1, Optional.empty()); + assertSame(uris0, uris1); + + ArchiveUris uris2 = uris0.with(t1, Optional.of("scheme://test123")); + assertEquals(Map.of(t1, "scheme://test123/"), uris2.tenantArchiveUris()); + + assertSame(uris2, uris2.with(t1, Optional.of("scheme://test123"))); + assertSame(uris2, uris2.with(t1, Optional.of("scheme://test123/"))); + assertEquals(uris0, uris2.with(t1, Optional.empty())); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java new file mode 100644 index 00000000000..6204ace8a51 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class ArchiveUriSerializerTest { + + @Test + public void test_serialization() { + ArchiveUris archiveUris = new ArchiveUris(Map.of( + TenantName.from("tenant1"), "ftp://host123.test/dir/", + TenantName.from("tenant2"), "ftp://archive.test/vespa/"), + Map.of(CloudAccount.from("321456987012"), "ftp://host123.test/dir/")); + + ArchiveUris serialized = ArchiveUriSerializer.fromJson(ArchiveUriSerializer.toJson(archiveUris)); + assertEquals(archiveUris, serialized); + } + +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java deleted file mode 100644 index 2ae4f6363e0..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.config.provision.TenantName; -import org.junit.Test; - -import java.util.Map; -import java.util.TreeMap; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class TenantArchiveUriSerializerTest { - - @Test - public void test_serialization() { - Map<TenantName, String> archiveUris = new TreeMap<>(); - archiveUris.put(TenantName.from("tenant1"), "ftp://host123.test/dir/"); - archiveUris.put(TenantName.from("tenant2"), "ftp://archive.test/vespa/"); - - Map<TenantName, String> serialized = TenantArchiveUriSerializer.fromJson(TenantArchiveUriSerializer.toJson(archiveUris)); - assertEquals(archiveUris, serialized); - } - -}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java deleted file mode 100644 index 7751e906c48..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.node.Generation; -import org.junit.Test; - -import java.util.Optional; - -import static com.yahoo.vespa.hosted.provision.provisioning.ArchiveUris.normalizeUri; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; - -/** - * @author freva - */ -public class ArchiveUrisTest { - - @Test - public void archive_uri() { - ApplicationId app = ApplicationId.from("vespa", "music", "main"); - Node allocated = createNode(app); - Node unallocated = createNode(null); - ArchiveUris archiveUris = new ProvisioningTester.Builder().build().nodeRepository().archiveUris(); - - assertFalse(archiveUris.archiveUriFor(unallocated).isPresent()); - assertFalse(archiveUris.archiveUriFor(allocated).isPresent()); - - archiveUris.setArchiveUri(app.tenant(), Optional.of("scheme://hostname/dir")); - assertEquals("scheme://hostname/dir/music/main/default/h432a/", archiveUris.archiveUriFor(allocated).get()); - } - - private Node createNode(ApplicationId appId) { - Node.Builder nodeBuilder = Node.create("id", "h432a.prod.us-south-1.vespa.domain.tld", new Flavor(NodeResources.unspecified()), Node.State.parked, NodeType.tenant); - Optional.ofNullable(appId) - .map(app -> new Allocation(app, - ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), - NodeResources.unspecified(), - Generation.initial(), - false)) - .ifPresent(nodeBuilder::allocation); - return nodeBuilder.build(); - } - - @Test - public void normalize_test() { - assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123")); - assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123/")); - assertEquals("s3://my-bucket-prod.region/my-tenant-123/", normalizeUri("s3://my-bucket-prod.region/my-tenant-123/")); - assertEquals("s3://my-bucket-prod.region/my-tenant_123/", normalizeUri("s3://my-bucket-prod.region/my-tenant_123/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("domain/dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain/dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain//dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal:dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/-illegal-dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/_illegal-dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir-/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir_/")); - } - - private static void assertThrows(Class<? extends Throwable> clazz, Runnable runnable) { - try { - runnable.run(); - fail("Expected " + clazz); - } catch (Throwable e) { - if (!clazz.isInstance(e)) throw e; - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java new file mode 100644 index 00000000000..80d79b036e2 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; +import com.yahoo.text.Utf8; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test of the REST APIs provided by the node repository. + * + * Note: This class is referenced from our operations documentation and must not be renamed/moved without updating that. + * + * @author bratseth + */ +public class ArchiveApiTest { + + private RestApiTester tester; + + @Before + public void createTester() { + tester = new RestApiTester(SystemName.Public, CloudAccount.from("111222333444")); + } + + @After + public void closeTester() { + tester.close(); + } + + @Test + public void archive_uris() throws IOException { + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive"), "{\"archives\":[]}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", Utf8.toBytes("{\"uri\": \"ftp://host/dir\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for tenant3\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant2", Utf8.toBytes("{\"uri\": \"s3://my-bucket/dir\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for tenant2\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/account/777888999000", Utf8.toBytes("{\"uri\": \"s3://acc-bucket\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for 777888999000\"}"); + + + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/application3/instance3/id3/host4/\"", true); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/node-admin/dockerhost2/\"", true); + assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/account/777888999000", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for 777888999000\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "archiveUri", false); + } + + + private void assertFile(Request request, String file) throws IOException { + tester.assertFile(request, file); + } + + private void assertResponse(Request request, String response) throws IOException { + tester.assertResponse(request, response); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java index 240d0daf96f..729b6b813cd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -16,7 +17,7 @@ public class LoadBalancersV1ApiTest { @Before public void createTester() { - tester = new RestApiTester(CloudAccount.empty); + tester = new RestApiTester(SystemName.main, CloudAccount.empty); } @After diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 43676518330..c9e57c22d11 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -6,6 +6,7 @@ import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; import com.yahoo.vespa.applicationmodel.HostName; @@ -41,7 +42,7 @@ public class NodesV2ApiTest { @Before public void createTester() { - tester = new RestApiTester(CloudAccount.from("111222333444")); + tester = new RestApiTester(SystemName.main, CloudAccount.from("111222333444")); } @After @@ -990,23 +991,6 @@ public class NodesV2ApiTest { } @Test - public void archive_uris() throws IOException { - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive"), "{\"archives\":[]}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant3", Utf8.toBytes("{\"uri\": \"ftp://host/dir\"}"), Request.Method.PATCH), - "{\"message\":\"Updated archive URI for tenant3\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant2", Utf8.toBytes("{\"uri\": \"s3://my-bucket/dir\"}"), Request.Method.PATCH), - "{\"message\":\"Updated archive URI for tenant2\"}"); - - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/application3/instance3/id3/host4/\"", true); - assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json"); - - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}"); - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); - } - - @Test public void trusted_certificates_patch() throws IOException { String url = "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"; tester.assertPartialResponse(new Request(url), "\"trustStore\":[]", false); // initially empty list diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java index e424b04aeaf..47745d25467 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java @@ -6,6 +6,7 @@ import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; import com.yahoo.io.IOUtils; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.ComparisonFailure; @@ -25,8 +26,8 @@ public class RestApiTester { private final JDisc container; - public RestApiTester(CloudAccount defaultCloudAccount) { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0, defaultCloudAccount), Networking.disable); + public RestApiTester(SystemName systemName, CloudAccount defaultCloudAccount) { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0, systemName, defaultCloudAccount), Networking.disable); } public void close() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json index 1ce54b54f6a..738d8ee1bb3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json @@ -7,6 +7,10 @@ { "tenant": "tenant3", "uri": "ftp://host/dir/" + }, + { + "account": "777888999000", + "uri": "s3://acc-bucket/" } ] } |