diff options
Diffstat (limited to 'node-repository/src')
3 files changed, 104 insertions, 32 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 6582c03d727..aae64d0563c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -60,6 +60,7 @@ public class NodeRepository extends AbstractComponent { private final JobControl jobControl; private final Applications applications; private final LoadBalancers loadBalancers; + private final FlagSource flagSource; private final int spareCount; /** @@ -102,11 +103,10 @@ public class NodeRepository extends AbstractComponent { boolean useCuratorClientCache, int spareCount, long nodeCacheSize) { - // TODO (valerijf): Uncomment when exception for prod.cd-aws is removed -// if (provisionServiceProvider.getHostProvisioner().isPresent() != zone.getCloud().dynamicProvisioning()) -// throw new IllegalArgumentException(String.format( -// "dynamicProvisioning property must be 1-to-1 with availability of HostProvisioner, was: dynamicProvisioning=%s, hostProvisioner=%s", -// zone.getCloud().dynamicProvisioning(), provisionServiceProvider.getHostProvisioner().map(__ -> "present").orElse("empty"))); + if (provisionServiceProvider.getHostProvisioner().isPresent() != zone.getCloud().dynamicProvisioning()) + throw new IllegalArgumentException(String.format( + "dynamicProvisioning property must be 1-to-1 with availability of HostProvisioner, was: dynamicProvisioning=%s, hostProvisioner=%s", + zone.getCloud().dynamicProvisioning(), provisionServiceProvider.getHostProvisioner().map(__ -> "present").orElse("empty"))); this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, nodeCacheSize); this.zone = zone; @@ -122,6 +122,7 @@ public class NodeRepository extends AbstractComponent { this.jobControl = new JobControl(new JobControlFlags(db, flagSource)); this.applications = new Applications(db); this.loadBalancers = new LoadBalancers(db); + this.flagSource = flagSource; this.spareCount = spareCount; rewriteNodes(); } @@ -174,6 +175,14 @@ public class NodeRepository extends AbstractComponent { public HostResourcesCalculator resourcesCalculator() { return resourcesCalculator; } + public FlagSource flagSource() { return flagSource; } + + /** Returns the time keeper of this system */ + public Clock clock() { return clock; } + + /** Returns the zone of this system */ + public Zone zone() { return zone; } + /** The number of nodes we should ensure has free capacity for node failures whenever possible */ public int spareCount() { return spareCount; } @@ -204,10 +213,4 @@ public class NodeRepository extends AbstractComponent { applications.remove(transaction); } - /** Returns the time keeper of this system */ - public Clock clock() { return clock; } - - /** Returns the zone of this system */ - public Zone zone() { return zone; } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 502de3ef734..eaef044e07c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -8,11 +8,12 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Address; @@ -22,8 +23,6 @@ import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; -import java.io.IOException; -import java.io.OutputStream; import java.net.URI; import java.util.List; import java.util.Optional; @@ -33,7 +32,7 @@ import java.util.function.Function; /** * @author bratseth */ -class NodesResponse extends HttpResponse { +class NodesResponse extends SlimeJsonResponse { /** The responses this can create */ public enum ResponseType { nodeList, stateList, nodesInStateList, singleNode } @@ -48,19 +47,16 @@ class NodesResponse extends HttpResponse { private final boolean recursive; private final Function<HostName, Optional<HostInfo>> orchestrator; private final NodeRepository nodeRepository; - private final Slime slime; - public NodesResponse(ResponseType responseType, HttpRequest request, + public NodesResponse(ResponseType responseType, HttpRequest request, Orchestrator orchestrator, NodeRepository nodeRepository) { - super(200); this.parentUrl = toParentUrl(request); this.nodeParentUrl = toNodeParentUrl(request); - filter = NodesV2ApiHandler.toNodeFilter(request); + this.filter = NodesV2ApiHandler.toNodeFilter(request); this.recursive = request.getBooleanProperty("recursive"); this.orchestrator = orchestrator.getHostResolver(); this.nodeRepository = nodeRepository; - slime = new Slime(); Cursor root = slime.setObject(); switch (responseType) { case nodeList: nodesToSlime(root); break; @@ -84,16 +80,6 @@ class NodesResponse extends HttpResponse { return uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + "/nodes/v2/node/"; } - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { - return "application/json"; - } - private void statesToSlime(Cursor root) { Cursor states = root.setObject("states"); for (Node.State state : Node.State.values()) @@ -193,6 +179,7 @@ class NodesResponse extends HttpResponse { node.reports().toSlime(object, "reports"); node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); + archiveUri(nodeRepository.flagSource(), node).ifPresent(uri -> object.setString("archiveUri", uri)); } private void toSlime(ApplicationId id, Cursor object) { @@ -247,4 +234,30 @@ class NodesResponse extends HttpResponse { return path.substring(lastSlash+1); } + // TODO (freva): Store this in Application or Node + static Optional<String> archiveUri(FlagSource flagSource, Node node) { + String bucket = Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.bindTo(flagSource) + .with(FetchVector.Dimension.NODE_TYPE, node.type().name()) + .with(FetchVector.Dimension.APPLICATION_ID, node.allocation().map(alloc -> alloc.owner().serializedForm()).orElse(null)) + .value(); + if (bucket.isBlank()) return Optional.empty(); + + StringBuilder sb = new StringBuilder(100).append("s3://").append(bucket).append('/'); + if (node.type() == NodeType.tenant) { + if (node.allocation().isEmpty()) return Optional.empty(); + ApplicationId app = node.allocation().get().owner(); + + sb.append(app.tenant().value()).append('/').append(app.application().value()).append('/').append(app.instance().value()).append('/'); + } else { + sb.append("hosted-vespa/"); + } + + for (char c: node.hostname().toCharArray()) { + if (c == '.') break; + sb.append(c); + } + + return Optional.of(sb.append('/').toString()); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponseTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponseTest.java new file mode 100644 index 00000000000..021afd0df7c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponseTest.java @@ -0,0 +1,56 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class NodesResponseTest { + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + + @Test + public void archive_uri() { + ApplicationId app = ApplicationId.from("vespa", "music", "main"); + // Flag not set, no archive uri + assertArchiveUri(null, "h432a.prod.us-south-1.vespa.domain.tld", NodeType.tenant, app); + assertArchiveUri(null, "cfg1.prod.us-south-1.vespa.domain.tld", NodeType.config, app); + + flagSource.withStringFlag(Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.id(), "vespa-data-bucket"); + // Flag is set, but node not allocated, only sync non-tenant nodes + assertArchiveUri(null, "h432a.prod.us-south-1.vespa.domain.tld", NodeType.tenant, null); + assertArchiveUri("s3://vespa-data-bucket/hosted-vespa/cfg1/", "cfg1.prod.us-south-1.vespa.domain.tld", NodeType.config, null); + + // Flag is set and node is allocated + assertArchiveUri("s3://vespa-data-bucket/vespa/music/main/h432a/", "h432a.prod.us-south-1.vespa.domain.tld", NodeType.tenant, app); + assertArchiveUri("s3://vespa-data-bucket/hosted-vespa/cfg1/", "cfg1.prod.us-south-1.vespa.domain.tld", NodeType.config, app); + } + + private void assertArchiveUri(String archiveUri, String hostname, NodeType type, ApplicationId appId) { + Node.Builder nodeBuilder = Node.create("id", hostname, new Flavor(NodeResources.unspecified()), Node.State.parked, type); + Optional.ofNullable(appId) + .map(app -> new Allocation(app, + ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), + NodeResources.unspecified(), + Generation.initial(), + false)) + .ifPresent(nodeBuilder::allocation); + + assertEquals(archiveUri, NodesResponse.archiveUri(flagSource, nodeBuilder.build()).orElse(null)); + } +}
\ No newline at end of file |