diff options
| author | Jon Bratseth <bratseth@verizonmedia.com> | 2019-10-07 09:45:48 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@verizonmedia.com> | 2019-10-07 09:45:48 +0200 |
| commit | 8729925b15b81bd3a5d0a0835c631843cd791178 (patch) | |
| tree | c7e6dc8f3c2fb429644004f45429c8db5bf9e6ad | |
| parent | 3188f79fdad37e3ea30f84f8c3be67b0c645386d (diff) | |
| parent | 260e989c42beb61608f4e8ebbffbe54a59ef4602 (diff) | |
Merge with master
144 files changed, 2554 insertions, 1217 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java index 6d121657a40..447b6efb09b 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java @@ -35,14 +35,17 @@ public class Certificates { var now = clock.instant(); var notBefore = now.minus(Duration.ofHours(1)); var notAfter = now.plus(CERTIFICATE_TTL); - return X509CertificateBuilder.fromCsr(csr, + var builder = X509CertificateBuilder.fromCsr(csr, x500principal, notBefore, notAfter, caPrivateKey, SHA256_WITH_ECDSA, - X509CertificateBuilder.generateRandomSerialNumber()) - .build(); + X509CertificateBuilder.generateRandomSerialNumber()); + for (var san : csr.getSubjectAlternativeNames()) { + builder = builder.addSubjectAlternativeName(san.getValue()); + } + return builder.build(); } /** Returns the DNS name field from Subject Alternative Names in given csr */ diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java index 4e306d9a70e..80940dcd02c 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java @@ -3,26 +3,32 @@ package com.yahoo.vespa.hosted.ca; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SubjectAlternativeName; import com.yahoo.test.ManualClock; import org.junit.Test; +import java.security.KeyPair; +import java.security.cert.X509Certificate; import java.time.Duration; +import java.util.List; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** * @author mpolden */ public class CertificatesTest { + private final KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private final X509Certificate caCertificate = CertificateTester.createCertificate("CA", keyPair); + @Test public void expiry() { var clock = new ManualClock(); var certificates = new Certificates(clock); var csr = CertificateTester.createCsr(); - var keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - var caCertificate = CertificateTester.createCertificate("CA", keyPair); var certificate = certificates.create(csr, caCertificate, keyPair.getPrivate()); var now = clock.instant(); @@ -30,4 +36,17 @@ public class CertificatesTest { assertEquals(now.plus(Duration.ofDays(30)).truncatedTo(SECONDS), certificate.getNotAfter().toInstant()); } + @Test + public void add_san_from_csr() throws Exception { + var certificates = new Certificates(new ManualClock()); + var dnsName = "host.example.com"; + var csr = CertificateTester.createCsr(dnsName); + var certificate = certificates.create(csr, caCertificate, keyPair.getPrivate()); + + assertNotNull(certificate.getSubjectAlternativeNames()); + assertEquals(1, certificate.getSubjectAlternativeNames().size()); + assertEquals(List.of(SubjectAlternativeName.Type.DNS_NAME.getTag(), dnsName), + certificate.getSubjectAlternativeNames().iterator().next()); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 09e67ed96cb..e579f736136 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -55,7 +55,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { } if (! hostname.contains(".")) { deployLogger.log(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + - "since it is not a canonical hostname." + + "since it is not a canonical hostname. " + "Disregard this warning when testing in a Docker container."); } } diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 92407aa9c68..05de643a116 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -89,7 +89,6 @@ <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> - <include>commons-daemon:commons-daemon:[${commons-daemon.version}]:jar:provided</include> <include>commons-logging:commons-logging:[1.1.1]:jar:provided</include> <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 5fc148beaee..6b52c5023fc 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -115,11 +115,6 @@ <classifier>no_aop</classifier> </dependency> <dependency> - <groupId>commons-daemon</groupId> - <artifactId>commons-daemon</artifactId> - <version>${commons-daemon.version}</version> - </dependency> - <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> <!-- This version is exported by jdisc via jcl-over-slf4j. --> @@ -442,7 +437,6 @@ <properties> <aopalliance.version>1.0</aopalliance.version> <bouncycastle.version>1.63</bouncycastle.version> - <commons-daemon.version>1.0.3</commons-daemon.version> <felix.version>6.0.3</felix.version> <felix.log.version>1.0.1</felix.log.version> <findbugs.version>1.3.9</findbugs.version> diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 1f621eb926c..0c8e564578b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -7,7 +7,6 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.jdisc.Metric; -import com.yahoo.net.HostName; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; @@ -27,8 +26,6 @@ import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; import org.apache.commons.lang.StringUtils; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -371,6 +368,10 @@ public class ClusterSearcher extends Searcher { } @Override - public void deconstruct() { } + public void deconstruct() { + if (server != null) { + server.shutDown(); + } + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index b0b3a7800e9..9a4913b3840 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -173,4 +173,9 @@ public class FastSearcher extends VespaBackEndSearcher { return getLogger().isLoggable(Level.FINE); } + @Override + public void shutDown() { + super.shutDown(); + dispatcher.shutDown(); + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 8f4b49ac71e..bc3ac6cdef1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -392,4 +392,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return getLogger().isLoggable(Level.FINE); } + public void shutDown() { } + } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 22c7f59872c..a016f7d695c 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -9,7 +9,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,9 +28,9 @@ public class ClusterMonitor<T> { private static Logger log = Logger.getLogger(ClusterMonitor.class.getName()); - private NodeManager<T> nodeManager; + private final NodeManager<T> nodeManager; - private MonitorThread monitorThread; + private final MonitorThread monitorThread; private volatile boolean shutdown = false; @@ -119,28 +121,35 @@ public class ClusterMonitor<T> { } public void run() { - log.fine("Starting cluster monitor thread"); + log.info("Starting cluster monitor thread " + getName()); // Pings must happen in a separate thread from this to handle timeouts // By using a cached thread pool we ensured that 1) a single thread will be used // for all pings when there are no problems (important because it ensures that // any thread local connections are reused) 2) a new thread will be started to execute // new pings when a ping is not responding - Executor pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping")); + ExecutorService pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping")); while (!isInterrupted()) { try { Thread.sleep(configuration.getCheckInterval()); log.finest("Activating ping"); ping(pingExecutor); } - catch (Exception e) { + catch (Throwable e) { if (shutdown && e instanceof InterruptedException) { break; + } else if ( ! (e instanceof Exception) ) { + log.log(Level.WARNING,"Error in monitor thread, will quit", e); + break; } else { - log.log(Level.WARNING,"Error in monitor thread",e); + log.log(Level.WARNING,"Exception in monitor thread", e); } } } - log.fine("Stopped cluster monitor thread"); + pingExecutor.shutdown(); + try { + pingExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { } + log.info("Stopped cluster monitor thread " + getName()); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 7369b33e82d..ddd319b7bcb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -195,6 +195,10 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } + public void shutDown() { + searchCluster.shutDown(); + } + private void emitDispatchMetric(Optional<SearchInvoker> invoker) { if (invoker.isEmpty()) { metric.add(FDISPATCH_METRIC, 1, metricContext); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index b47f2fefa5b..09ad715b471 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -16,17 +16,15 @@ public class Node { private final int key; private int pathIndex; private final String hostname; - private final int fs4port; - final int group; + private final int group; private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); - public Node(int key, String hostname, int fs4port, int group) { + public Node(int key, String hostname, int group) { this.key = key; this.hostname = hostname; - this.fs4port = fs4port; this.group = group; } @@ -41,14 +39,15 @@ public class Node { public String hostname() { return hostname; } - public int fs4port() { return fs4port; } - /** Returns the id of this group this node belongs to */ public int group() { return group; } public void setWorking(boolean working) { this.statusIsKnown.lazySet(true); this.working.lazySet(working); + if ( ! working ) { + activeDocuments.set(0); + } } /** Returns whether this node is currently responding to requests, or null if status is not known */ @@ -57,17 +56,17 @@ public class Node { } /** Updates the active documents on this node */ - public void setActiveDocuments(long activeDocuments) { + void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } /** Returns the active documents on this node. If unknown, 0 is returned. */ - public long getActiveDocuments() { - return this.activeDocuments.get(); + long getActiveDocuments() { + return activeDocuments.get(); } @Override - public int hashCode() { return Objects.hash(hostname, fs4port); } + public int hashCode() { return Objects.hash(hostname, key, pathIndex, group); } @Override public boolean equals(Object o) { @@ -75,11 +74,15 @@ public class Node { if ( ! (o instanceof Node)) return false; Node other = (Node)o; if ( ! Objects.equals(this.hostname, other.hostname)) return false; - if ( ! Objects.equals(this.fs4port, other.fs4port)) return false; + if ( ! Objects.equals(this.key, other.key)) return false; + if ( ! Objects.equals(this.pathIndex, other.pathIndex)) return false; + if ( ! Objects.equals(this.group, other.group)) return false; + return true; } @Override - public String toString() { return "search node " + hostname + ":" + fs4port + " in group " + group; } + public String toString() { return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + + " statusIsKnown = " + statusIsKnown.get() + " working = " + working.get() + " activeDocs = " + activeDocuments.get(); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index a55a970e8ff..3595a24ca92 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -46,6 +46,7 @@ public class SearchCluster implements NodeManager<Node> { private final ClusterMonitor<Node> clusterMonitor; private final VipStatus vipStatus; private PingFactory pingFactory; + private long nextLogTime = 0; /** * A search node on this local machine having the entire corpus, which we therefore @@ -73,7 +74,7 @@ public class SearchCluster implements NodeManager<Node> { } this.groups = groupsBuilder.build(); LinkedHashMap<Integer, Group> groupIntroductionOrder = new LinkedHashMap<>(); - nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group))); + nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group()))); this.orderedGroups = ImmutableList.<Group>builder().addAll(groupIntroductionOrder.values()).build(); // Index nodes by host @@ -91,6 +92,10 @@ public class SearchCluster implements NodeManager<Node> { this.clusterMonitor = new ClusterMonitor<>(this); } + public void shutDown() { + clusterMonitor.shutdown(); + } + public void startClusterMonitoring(PingFactory pingFactory) { this.pingFactory = pingFactory; @@ -141,7 +146,7 @@ public class SearchCluster implements NodeManager<Node> { } for (DispatchConfig.Node node : dispatchConfig.node()) { if (filter.test(node)) { - nodesBuilder.add(new Node(node.key(), node.host(), node.fs4port(), node.group())); + nodesBuilder.add(new Node(node.key(), node.host(), node.group())); } } return nodesBuilder.build(); @@ -409,14 +414,21 @@ public class SearchCluster implements NodeManager<Node> { private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { boolean changed = group.isFullCoverageStatusChanged(fullCoverage); - if (changed) { + if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { + nextLogTime = System.currentTimeMillis() + 30 * 1000; int requiredNodes = groupSize() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", index, group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); } else { - log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d)", - index, group.workingNodes(), groupSize(), requiredNodes)); + StringBuilder missing = new StringBuilder(); + for (var node : group.nodes()) { + if (node.isWorking() != Boolean.TRUE) { + missing.append('\n').append(node.toString()); + } + } + log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", + index, group.workingNodes(), groupSize(), requiredNodes, group.getActiveDocuments(), averageDocuments, missing.toString())); } } } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index eb4d65693bb..4011611b049 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -81,7 +81,7 @@ public class FastSearcherTestCase { @Test public void testSinglePassGroupingIsForcedWithSingleNodeGroups() { FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 123, 0))), + MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), documentdbInfoConfig); @@ -102,7 +102,7 @@ public class FastSearcherTestCase { @Test public void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { - MockDispatcher dispatcher = MockDispatcher.create(ImmutableList.of(new Node(0, "host0", 123, 0), new Node(2, "host1", 123, 0))); + MockDispatcher dispatcher = MockDispatcher.create(ImmutableList.of(new Node(0, "host0", 0), new Node(2, "host1", 0))); FastSearcher fastSearcher = new FastSearcher("container.0", dispatcher, diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 0aa91442712..4fbbd9dd936 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -39,7 +39,6 @@ class MockDispatcher extends Dispatcher { for (Node node : nodes) { DispatchConfig.Node.Builder dispatchConfigNodeBuilder = new DispatchConfig.Node.Builder(); dispatchConfigNodeBuilder.host(node.hostname()); - dispatchConfigNodeBuilder.fs4port(node.fs4port()); dispatchConfigNodeBuilder.port(0); // Mandatory, but currently not used here dispatchConfigNodeBuilder.group(node.group()); dispatchConfigNodeBuilder.key(key++); // not used diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 310f536f961..3d544f5c114 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -70,7 +70,7 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 0, 0) { @Override public Optional<Node> localCorpusDispatchTarget() { - return Optional.of(new Node(1, "test", 123, 1)); + return Optional.of(new Node(1, "test", 1)); } }; MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 1ebf7940f25..0496194f8ed 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -28,7 +28,7 @@ import static org.junit.Assert.assertThat; public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { - Node n1 = new Node(0, "test-node1", 0, 0); + Node n1 = new Node(0, "test-node1", 0); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -41,8 +41,8 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesMultiGroupSetups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -55,10 +55,10 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesClusteredGroups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 0); - Node n3 = new Node(0, "test-node3", 0, 1); - Node n4 = new Node(1, "test-node4", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 0); + Node n3 = new Node(0, "test-node3", 1); + Node n4 = new Node(1, "test-node4", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -68,8 +68,8 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerReturnsDifferentGroups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index 2fe434d6f3f..c5fbda7c2f5 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -19,7 +19,7 @@ class MockInvoker extends SearchInvoker { private List<Hit> hits; protected MockInvoker(int key, Coverage coverage) { - super(Optional.of(new Node(key, "?", 0, 0))); + super(Optional.of(new Node(key, "?", 0))); this.coverage = coverage; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index e3ff54102d4..0bcc30d9b10 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -38,7 +38,7 @@ public class MockSearchCluster extends SearchCluster { for (int group = 0; group < groups; group++) { List<Node> nodes = new ArrayList<>(); for (int node = 0; node < nodesPerGroup; node++) { - Node n = new Node(dk, "host" + dk, -1, group); + Node n = new Node(dk, "host" + dk, group); n.setWorking(true); nodes.add(n); hostBuilder.put(n.hostname(), n); @@ -124,8 +124,9 @@ public class MockSearchCluster extends SearchCluster { builder.minWaitAfterCoverageFactor(0); builder.maxWaitAfterCoverageFactor(0.5); } + int port = 10000; for (Node n : nodes) { - builder.node(new DispatchConfig.Node.Builder().key(n.key()).host(n.hostname()).port(n.fs4port()).group(n.group())); + builder.node(new DispatchConfig.Node.Builder().key(n.key()).host(n.hostname()).port(port++).group(n.group())); } return new DispatchConfig(builder); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index d629bd36bb1..c07bf119782 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -36,7 +36,7 @@ public class RpcSearchInvokerTest { var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); @SuppressWarnings("resource") - var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 77, 1), mockPool); + var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool); Query q = new Query("search/?query=test&hits=10&offset=3"); invoker.sendSearchRequest(q); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index f29d6ddf324..f42185e955f 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -63,7 +63,7 @@ public class SearchClusterTest { for (String name : nodeNames) { int key = nodes.size() % nodesPergroup; int group = nodes.size() / nodesPergroup; - nodes.add(new Node(key, name, 13333, group)); + nodes.add(new Node(key, name, group)); numDocsPerNode.add(new AtomicInteger(1)); pingCounts.add(new AtomicInteger(0)); } @@ -132,7 +132,7 @@ public class SearchClusterTest { @Override public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { - int index = node.group*numPerGroup + node.key(); + int index = node.group() * numPerGroup + node.key(); return new Pinger(activeDocs.get(index), pingCounts.get(index)); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java index fceecedb9fe..c2512c2032b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java @@ -10,10 +10,6 @@ public class ApplicationId extends NonDefaultIdentifier { super(id); } - public static boolean isLegal(String id) { - return strictPattern.matcher(id).matches(); - } - @Override public void validate() { super.validate(); @@ -21,9 +17,8 @@ public class ApplicationId extends NonDefaultIdentifier { } public static void validate(String id) { - if (!isLegal(id)) { + if ( ! strictPattern.matcher(id).matches()) throwInvalidId(id, strictPatternExplanation); - } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java index 2067a88e5fb..4007ac2b9cd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java @@ -15,7 +15,6 @@ public abstract class Identifier { protected static final String strictPatternExplanation = "New tenant or application names must start with a letter, may contain no more than 20 " + "characters, and may only contain lowercase letters, digits or dashes, but no double-dashes."; - // TODO: Use this also for instances, if they ever get proper support. protected static final Pattern strictPattern = Pattern.compile("^(?=.{1,20}$)[a-z](-?[a-z0-9]+)*$"); private static final Pattern serializedIdentifierPattern = Pattern.compile("[a-zA-Z0-9_-]+"); private static final Pattern serializedPattern = Pattern.compile("[a-zA-Z0-9_.-]+"); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java index 6e3087cdcf6..8e14774b827 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java @@ -16,4 +16,9 @@ public class InstanceId extends SerializedIdentifier { validateNoUpperCase(); } + public static void validate(String id) { + if ( ! strictPattern.matcher(id).matches()) + throwInvalidId(id, strictPatternExplanation); + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index f36107db228..606db8a0f2f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -58,6 +58,26 @@ public abstract class Role { return new TenantRole(RoleDefinition.tenantOperator, tenant); } + /** Returns a {@link RoleDefinition#reader} for the current system and given tenant. */ + public static TenantRole reader(TenantName tenant) { + return new TenantRole(RoleDefinition.reader, tenant); + } + + /** Returns a {@link RoleDefinition#developer} for the current system and given tenant. */ + public static TenantRole developer(TenantName tenant) { + return new TenantRole(RoleDefinition.developer, tenant); + } + + /** Returns a {@link RoleDefinition#administrator} for the current system and given tenant. */ + public static TenantRole administrator(TenantName tenant) { + return new TenantRole(RoleDefinition.administrator, tenant); + } + + /** Returns a {@link RoleDefinition#headless} for the current system, given tenant, and application */ + public static ApplicationRole headless(TenantName tenant, ApplicationName application) { + return new ApplicationRole(RoleDefinition.headless, tenant, application); + } + /** Returns a {@link RoleDefinition#applicationAdmin} for the current system and given tenant and application. */ public static ApplicationRole applicationAdmin(TenantName tenant, ApplicationName application) { return new ApplicationRole(RoleDefinition.applicationAdmin, tenant, application); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index 7bbd89404c7..8e3754777ea 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -70,6 +70,29 @@ public enum RoleDefinition { tenantOwner(tenantAdmin, Policy.tenantDelete), + /** Reader — the base role for all tenant users */ + reader(Policy.tenantRead, + Policy.applicationRead, + Policy.deploymentRead, + Policy.publicRead), + + /** User — the dev.ops. role for normal Vespa tenant users */ + developer(Policy.applicationCreate, + Policy.applicationUpdate, + Policy.applicationDelete, + Policy.applicationOperations, + Policy.developmentDeployment, + Policy.keyManagement, + Policy.submission), + + /** Admin — the administrative function for user management etc. */ + administrator(Policy.tenantUpdate, + Policy.tenantManager, + Policy.applicationManager), + + /** Headless — the application specific role identified by deployment keys for production */ + headless(Policy.submission), + /** Build and continuous delivery service. */ // TODO replace with buildService, when everyone is on new pipeline. tenantPipeline(everyone, Policy.submission, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index c17ac044136..c83f366cb67 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import java.security.PublicKey; import java.time.Instant; import java.util.Collection; import java.util.Comparator; @@ -51,7 +52,7 @@ public class Application { private final Optional<User> owner; private final OptionalInt majorVersion; private final ApplicationMetrics metrics; - private final Set<String> pemDeployKeys; + private final Set<PublicKey> deployKeys; private final Map<InstanceName, Instance> instances; /** Creates an empty application. */ @@ -64,7 +65,7 @@ public class Application { // DO NOT USE! For serialization purposes, only. public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Change change, Change outstandingChange, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, - OptionalInt majorVersion, ApplicationMetrics metrics, Set<String> pemDeployKeys, + OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, OptionalLong projectId, boolean internal, Collection<Instance> instances) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); @@ -77,7 +78,7 @@ public class Application { this.owner = Objects.requireNonNull(owner, "owner cannot be null"); this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null"); this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); - this.pemDeployKeys = Objects.requireNonNull(pemDeployKeys, "pemDeployKeys cannot be null"); + this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null"); this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null"); this.internal = internal; this.instances = ImmutableSortedMap.copyOf(instances.stream().collect(Collectors.toMap(Instance::name, Function.identity()))); @@ -191,7 +192,7 @@ public class Application { } /** Returns the set of deploy keys for this application. */ - public Set<String> pemDeployKeys() { return pemDeployKeys; } + public Set<PublicKey> deployKeys() { return deployKeys; } @Override public boolean equals(Object o) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 6f64237b2c4..0cf0f59102e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -2,12 +2,12 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; @@ -149,7 +150,6 @@ public class ApplicationController { // Update serialization format of all applications Once.after(Duration.ofMinutes(1), () -> { - curator.deleteOldApplicationData(); Instant start = clock.instant(); int count = 0; for (Application application : curator.readApplications()) { @@ -213,6 +213,14 @@ public class ApplicationController { public ApplicationStore applicationStore() { return applicationStore; } + /** Returns all content clusters in all current deployments of the given application. */ + public Map<ZoneId, List<String>> contentClustersByZone(ApplicationId id, Iterable<ZoneId> zones) { + ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder(); + for (ZoneId zone : zones) + clusters.put(zone, ImmutableList.copyOf(configServer.getContentClusters(new DeploymentId(id, zone)))); + return clusters.build(); + } + /** Returns the oldest Vespa version installed on any active or reserved production node for the given application. */ public Version oldestInstalledPlatform(TenantAndApplicationId id) { return requireApplication(id).instances().values().stream() @@ -263,70 +271,86 @@ public class ApplicationController { * * @throws IllegalArgumentException if the application already exists */ - // TODO jonmv: split in create application and create instance - public Application createApplication(ApplicationId id, Optional<Credentials> credentials) { - if (id.instance().isTester()) - throw new IllegalArgumentException("'" + id + "' is a tester application!"); - try (Lock lock = lock(TenantAndApplicationId.from(id))) { - // Validate only application names which do not already exist. - if (getApplication(TenantAndApplicationId.from(id)).isEmpty()) - com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value()); + public Application createApplication(TenantAndApplicationId id, Optional<Credentials> credentials) { + try (Lock lock = lock(id)) { + if (getApplication(id).isPresent()) + throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); + if (getApplication(dashToUnderscore(id)).isPresent()) // VESPA-1945 + throw new IllegalArgumentException("Could not create '" + id + "': Application " + dashToUnderscore(id) + " already exists"); + + com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value()); Optional<Tenant> tenant = controller.tenants().get(id.tenant()); if (tenant.isEmpty()) throw new IllegalArgumentException("Could not create '" + id + "': This tenant does not exist"); - if (getInstance(id).isPresent()) - throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); - if (getInstance(dashToUnderscore(id)).isPresent()) // VESPA-1945 - throw new IllegalArgumentException("Could not create '" + id + "': Application " + dashToUnderscore(id) + " already exists"); if (tenant.get().type() != Tenant.Type.user) { if (credentials.isEmpty()) throw new IllegalArgumentException("Could not create '" + id + "': No credentials provided"); - - if ( ! id.instance().isTester()) // Only store the application permits for non-user applications. - accessControl.createApplication(id, credentials.get()); + accessControl.createApplication(id, credentials.get()); } - Application application = getApplication(TenantAndApplicationId.from(id)).orElse(new Application(TenantAndApplicationId.from(id), - clock.instant())); - LockedApplication locked = new LockedApplication(application, lock).withNewInstance(id.instance()); + + LockedApplication locked = new LockedApplication(new Application(id, clock.instant()), lock); store(locked); log.info("Created " + locked); return locked.get(); } } + /** + * Creates a new instance for an existing application. + * + * @throws IllegalArgumentException if the instance already exists, or has an invalid instance name. + */ + public void createInstance(ApplicationId id) { + if (id.instance().isTester()) + throw new IllegalArgumentException("'" + id + "' is a tester application!"); + lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { + InstanceId.validate(id.instance().value()); + + if (getInstance(id).isPresent()) + throw new IllegalArgumentException("Could not create '" + id + "': Instance already exists"); + if (getInstance(dashToUnderscore(id)).isPresent()) // VESPA-1945 + throw new IllegalArgumentException("Could not create '" + id + "': Instance " + dashToUnderscore(id) + " already exists"); + + store(application.withNewInstance(id.instance())); + log.info("Created " + id); + }); + } + public ActivateResult deploy(ApplicationId applicationId, ZoneId zone, Optional<ApplicationPackage> applicationPackageFromDeployer, DeployOptions options) { - return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options, Optional.empty()); + return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options); } /** Deploys an application. If the application does not exist it is created. */ // TODO: Get rid of the options arg - // TODO(jvenstad): Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter. - public ActivateResult deploy(ApplicationId applicationId, ZoneId zone, + // TODO jonmv: Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter. + public ActivateResult deploy(ApplicationId instanceId, ZoneId zone, Optional<ApplicationPackage> applicationPackageFromDeployer, Optional<ApplicationVersion> applicationVersionFromDeployer, - DeployOptions options, - Optional<Principal> deployingIdentity) { - if (applicationId.instance().isTester()) - throw new IllegalArgumentException("'" + applicationId + "' is a tester application!"); - - // TODO jonmv: Change this to create instances on demand. - Tenant tenant = controller.tenants().require(applicationId.tenant()); - if (tenant.type() == Tenant.Type.user && getInstance(applicationId).isEmpty()) + DeployOptions options) { + if (instanceId.instance().isTester()) + throw new IllegalArgumentException("'" + instanceId + "' is a tester application!"); + + TenantAndApplicationId applicationId = TenantAndApplicationId.from(instanceId); + if ( getApplication(applicationId).isEmpty() + && controller.tenants().require(instanceId.tenant()).type() == Tenant.Type.user) createApplication(applicationId, Optional.empty()); - try (Lock deploymentLock = lockForDeployment(applicationId, zone)) { + if (getInstance(instanceId).isEmpty()) + createInstance(instanceId); + + try (Lock deploymentLock = lockForDeployment(instanceId, zone)) { Version platformVersion; ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; Set<ContainerEndpoint> endpoints; Optional<ApplicationCertificate> applicationCertificate; - try (Lock lock = lock(TenantAndApplicationId.from(applicationId))) { - LockedApplication application = new LockedApplication(requireApplication(TenantAndApplicationId.from(applicationId)), lock); - InstanceName instance = applicationId.instance(); + try (Lock lock = lock(applicationId)) { + LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); + InstanceName instance = instanceId.instance(); boolean manuallyDeployed = options.deployDirectly || zone.environment().isManuallyDeployed(); boolean preferOldestVersion = options.deployCurrentVersion; @@ -348,25 +372,22 @@ public class ApplicationController { if ( job.isEmpty() || job.get().lastTriggered().isEmpty() || job.get().lastCompleted().isPresent() && job.get().lastCompleted().get().at().isAfter(job.get().lastTriggered().get().at())) - return unexpectedDeployment(applicationId, zone); + return unexpectedDeployment(instanceId, zone); JobRun triggered = job.get().lastTriggered().get(); platformVersion = preferOldestVersion ? triggered.sourcePlatform().orElse(triggered.platform()) : triggered.platform(); applicationVersion = preferOldestVersion ? triggered.sourceApplication().orElse(triggered.application()) : triggered.application(); - applicationPackage = getApplicationPackage(applicationId, application.get().internal(), applicationVersion); - applicationPackage = withTesterCertificate(applicationPackage, applicationId, jobType); + applicationPackage = getApplicationPackage(instanceId, application.get().internal(), applicationVersion); + applicationPackage = withTesterCertificate(applicationPackage, instanceId, jobType); validateRun(application.get(), instance, zone, platformVersion, applicationVersion); } - // TODO jonmv: Remove this when all packages are validated upon submission, as in ApplicationApiHandler.submit(...). - verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity); - if (zone.environment().isProduction()) // Assign and register endpoints application = withRotation(applicationPackage.deploymentSpec(), application, instance); - endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(applicationId.instance()), zone); + endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { // Provisions a new certificate if missing @@ -385,11 +406,11 @@ public class ApplicationController { // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); - ActivateResult result = deploy(applicationId, applicationPackage, zone, options, endpoints, + ActivateResult result = deploy(instanceId, applicationPackage, zone, options, endpoints, applicationCertificate.orElse(null)); - lockApplicationOrThrow(TenantAndApplicationId.from(applicationId), application -> - store(application.with(applicationId.instance(), + lockApplicationOrThrow(applicationId, application -> + store(application.with(instanceId.instance(), instance -> instance.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), warningsFrom(result))))); return result; @@ -702,22 +723,24 @@ public class ApplicationController { * * @throws IllegalArgumentException if the application has deployments or the caller is not authorized */ - public void deleteApplication(TenantName tenantName, ApplicationName applicationName, Optional<Credentials> credentials) { - Tenant tenant = controller.tenants().require(tenantName); + public void deleteApplication(TenantAndApplicationId id, Optional<Credentials> credentials) { + Tenant tenant = controller.tenants().require(id.tenant()); if (tenant.type() != Tenant.Type.user && credentials.isEmpty()) - throw new IllegalArgumentException("Could not delete application '" + tenantName + "." + applicationName + "': No credentials provided"); + throw new IllegalArgumentException("Could not delete application '" + id + "': No credentials provided"); // Find all instances of the application - TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); List<ApplicationId> instances = requireApplication(id).instances().keySet().stream() .map(id::instance) .collect(Collectors.toUnmodifiableList()); if (instances.size() > 1) throw new IllegalArgumentException("Could not delete application; more than one instance present: " + instances); - // TODO: Make this one transaction when database is moved to ZooKeeper for (ApplicationId instance : instances) - deleteInstance(instance, credentials); + deleteInstance(instance); + + if (tenant.type() != Tenant.Type.user) + accessControl.deleteApplication(id, credentials.get()); + curator.removeApplication(id); } /** @@ -726,24 +749,20 @@ public class ApplicationController { * @throws IllegalArgumentException if the application has deployments or the caller is not authorized * @throws NotExistsException if the instance does not exist */ - public void deleteInstance(ApplicationId applicationId, Optional<Credentials> credentials) { - Tenant tenant = controller.tenants().require(applicationId.tenant()); - if (tenant.type() != Tenant.Type.user && credentials.isEmpty()) - throw new IllegalArgumentException("Could not delete application '" + applicationId + "': No credentials provided"); - - if (getInstance(applicationId).isEmpty()) - throw new NotExistsException("Could not delete application '" + applicationId + "': Application not found"); + public void deleteInstance(ApplicationId instanceId) { + if (getInstance(instanceId).isEmpty()) + throw new NotExistsException("Could not delete instance '" + instanceId + "': Instance not found"); - lockApplicationOrThrow(TenantAndApplicationId.from(applicationId), application -> { - if ( ! application.get().require(applicationId.instance()).deployments().isEmpty()) + lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { + if ( ! application.get().require(instanceId.instance()).deployments().isEmpty()) throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments in: " + - application.get().require(applicationId.instance()).deployments().keySet().stream().map(ZoneId::toString) + application.get().require(instanceId.instance()).deployments().keySet().stream().map(ZoneId::toString) .sorted().collect(Collectors.joining(", "))); - applicationStore.removeAll(applicationId); - applicationStore.removeAll(TesterId.of(applicationId)); + applicationStore.removeAll(instanceId); + applicationStore.removeAll(TesterId.of(instanceId)); - Instance instance = application.get().require(applicationId.instance()); + Instance instance = application.get().require(instanceId.instance()); instance.rotations().forEach(assignedRotation -> { var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()); endpoints.asList().stream() @@ -752,15 +771,10 @@ public class ApplicationController { controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal); }); }); - curator.storeWithoutInstance(application.without(applicationId.instance()).get()); + curator.writeApplication(application.without(instanceId.instance()).get()); - log.info("Deleted " + application); + log.info("Deleted " + instanceId); }); - - - if (tenant.type() != Tenant.Type.user && getApplication(applicationId).isEmpty()) - // TODO jonmv: Implementations ignore the instance — refactor to provide tenant and application names only. - accessControl.deleteApplication(applicationId, credentials.get()); } /** @@ -845,10 +859,12 @@ public class ApplicationController { public DeploymentTrigger deploymentTrigger() { return deploymentTrigger; } + private TenantAndApplicationId dashToUnderscore(TenantAndApplicationId id) { + return TenantAndApplicationId.from(id.tenant().value(), id.application().value().replaceAll("-", "_")); + } + private ApplicationId dashToUnderscore(ApplicationId id) { - return ApplicationId.from(id.tenant().value(), - id.application().value().replaceAll("-", "_"), - id.instance().value()); + return dashToUnderscore(TenantAndApplicationId.from(id)).instance(id.instance()); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 5aa5a8e13de..19921595dc2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -11,11 +11,10 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; +import java.security.PublicKey; import java.time.Instant; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -43,7 +42,7 @@ public class LockedApplication { private final Optional<User> owner; private final OptionalInt majorVersion; private final ApplicationMetrics metrics; - private final Set<String> pemDeployKeys; + private final Set<PublicKey> deployKeys; private final OptionalLong projectId; private final boolean internal; private final Map<InstanceName, Instance> instances; @@ -58,14 +57,14 @@ public class LockedApplication { this(Objects.requireNonNull(lock, "lock cannot be null"), application.id(), application.createdAt(), application.deploymentSpec(), application.validationOverrides(), application.change(), application.outstandingChange(), application.deploymentIssueId(), application.ownershipIssueId(), - application.owner(), application.majorVersion(), application.metrics(), application.pemDeployKeys(), + application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(), application.projectId(), application.internal(), application.instances()); } private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Change change, Change outstandingChange, Optional<IssueId> deploymentIssueId, Optional<IssueId> ownershipIssueId, Optional<User> owner, - OptionalInt majorVersion, ApplicationMetrics metrics, Set<String> pemDeployKeys, + OptionalInt majorVersion, ApplicationMetrics metrics, Set<PublicKey> deployKeys, OptionalLong projectId, boolean internal, Map<InstanceName, Instance> instances) { this.lock = lock; @@ -80,7 +79,7 @@ public class LockedApplication { this.owner = owner; this.majorVersion = majorVersion; this.metrics = metrics; - this.pemDeployKeys = pemDeployKeys; + this.deployKeys = deployKeys; this.projectId = projectId; this.internal = internal; this.instances = Map.copyOf(instances); @@ -89,7 +88,7 @@ public class LockedApplication { /** Returns a read-only copy of this */ public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances.values()); } @@ -97,7 +96,7 @@ public class LockedApplication { var instances = new HashMap<>(this.instances); instances.put(instance, new Instance(id.instance(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } @@ -105,7 +104,7 @@ public class LockedApplication { var instances = new HashMap<>(this.instances); instances.put(instance, modification.apply(instances.get(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } @@ -113,61 +112,61 @@ public class LockedApplication { var instances = new HashMap<>(this.instances); instances.remove(instance); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } public LockedApplication withBuiltInternally(boolean builtInternally) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, builtInternally, instances); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } public LockedApplication withChange(Change change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } public LockedApplication withOutstandingChange(Change outstandingChange) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys, projectId, internal, instances); } @@ -176,25 +175,25 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, pemDeployKeys, projectId, internal, instances); + metrics, deployKeys, projectId, internal, instances); } public LockedApplication with(ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKeys, + deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, internal, instances); } - public LockedApplication withPemDeployKey(String pemDeployKey) { - Set<String> keys = new LinkedHashSet<>(pemDeployKeys); + public LockedApplication withDeployKey(PublicKey pemDeployKey) { + Set<PublicKey> keys = new LinkedHashSet<>(deployKeys); keys.add(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, projectId, internal, instances); } - public LockedApplication withoutPemDeployKey(String pemDeployKey) { - Set<String> keys = new LinkedHashSet<>(pemDeployKeys); + public LockedApplication withoutDeployKey(PublicKey pemDeployKey) { + Set<PublicKey> keys = new LinkedHashSet<>(deployKeys); keys.remove(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index ecc8bd65b72..6caf716aed4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -2,8 +2,10 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; +import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; @@ -16,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import java.security.Principal; +import java.security.PublicKey; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -126,44 +129,39 @@ public abstract class LockedTenant { public static class Cloud extends LockedTenant { private final BillingInfo billingInfo; - private final BiMap<String, Principal> pemDeveloperKeys; + private final BiMap<PublicKey, Principal> developerKeys; - private Cloud(TenantName name, BillingInfo billingInfo, BiMap<String, Principal> pemDeveloperKeys) { + private Cloud(TenantName name, BillingInfo billingInfo, BiMap<PublicKey, Principal> developerKeys) { super(name); this.billingInfo = billingInfo; - this.pemDeveloperKeys = pemDeveloperKeys; + this.developerKeys = ImmutableBiMap.copyOf(developerKeys); } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.billingInfo(), tenant.pemDeveloperKeys()); + this(tenant.name(), tenant.billingInfo(), tenant.developerKeys()); } @Override public CloudTenant get() { - return new CloudTenant(name, billingInfo, pemDeveloperKeys); + return new CloudTenant(name, billingInfo, developerKeys); } public Cloud with(BillingInfo billingInfo) { - return new Cloud(name, billingInfo, pemDeveloperKeys); - } - - public Cloud withPemDeveloperKey(String pemKey, Principal principal) { - ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder(); - pemDeveloperKeys.forEach((key, user) -> { - if ( ! user.equals(principal)) - keys.put(key, user); - }); - keys.put(pemKey, principal); - return new Cloud(name, billingInfo, keys.build()); - } - - public Cloud withoutPemDeveloperKey(String pemKey) { - ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder(); - pemDeveloperKeys.forEach((key, user) -> { - if ( ! key.equals(pemKey)) - keys.put(key, user); - }); - return new Cloud(name, billingInfo, keys.build()); + return new Cloud(name, billingInfo, developerKeys); + } + + public Cloud withDeveloperKey(PublicKey key, Principal principal) { + BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); + if (keys.containsKey(key)) + throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key)); + keys.put(key, principal); + return new Cloud(name, billingInfo, keys); + } + + public Cloud withoutDeveloperKey(PublicKey key) { + BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); + keys.remove(key); + return new Cloud(name, billingInfo, keys); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 9df918e3f20..5ff564f7ad3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -63,7 +63,7 @@ public enum SystemApplication { .orElse(false); } - /** Returns the node types of this that should receive OS upgrades */ + /** Returns whether this should receive OS upgrades */ public boolean isEligibleForOsUpgrades() { return nodeType.isDockerHost(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java index 0b537535315..b4f0d6e2487 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/TenantAndApplicationId.java @@ -48,6 +48,10 @@ public class TenantAndApplicationId implements Comparable<TenantAndApplicationId return instance(InstanceName.defaultName()); } + public ApplicationId instance(String instance) { + return instance(InstanceName.from(instance)); + } + public ApplicationId instance(InstanceName instance) { return ApplicationId.from(tenant, application, instance); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 91f9e2d56d7..304a47044a1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.inject.Inject; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; @@ -15,10 +14,12 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.zms.RoleAction; import com.yahoo.vespa.athenz.client.zms.ZmsClient; +import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; @@ -142,7 +143,7 @@ public class AthenzFacade implements AccessControl { } @Override - public void createApplication(ApplicationId id, Credentials credentials) { + public void createApplication(TenantAndApplicationId id, Credentials credentials) { AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; createApplication(athenzCredentials.domain(), id.application(), athenzCredentials.token()); } @@ -152,11 +153,19 @@ public class AthenzFacade implements AccessControl { log("createProviderResourceGroup(" + "tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s, roleActions=%s)", domain, service.getDomain().getName(), service.getName(), application, tenantRoleActions); - zmsClient.createProviderResourceGroup(domain, service, application.value(), tenantRoleActions, token); + try { + zmsClient.createProviderResourceGroup(domain, service, application.value(), tenantRoleActions, token); + } + catch (ZmsClientException e) { + if (e.getErrorCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) + throw new ForbiddenException("Not authorized to create application", e); + else + throw e; + } } @Override - public void deleteApplication(ApplicationId id, Credentials credentials) { + public void deleteApplication(TenantAndApplicationId id, Credentials credentials) { AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; log("deleteProviderResourceGroup(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", athenzCredentials.domain(), service.getDomain().getName(), service.getName(), id.application()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 1828a189cad..50af8bd8611 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -1,8 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; @@ -468,7 +466,7 @@ public class InternalStepRunner implements StepRunner { testConfigSerializer.configJson(id.application(), id.type(), endpoints, - listClusters(id.application(), zones))); + controller.applications().contentClustersByZone(id.application(), zones))); return Optional.of(running); } @@ -690,14 +688,6 @@ public class InternalStepRunner implements StepRunner { throw new IllegalStateException("No step deploys to the zone this run is for!"); } - /** Returns all content clusters in all current deployments of the given real application. */ - private Map<ZoneId, List<String>> listClusters(ApplicationId id, Iterable<ZoneId> zones) { - ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder(); - for (ZoneId zone : zones) - clusters.put(zone, ImmutableList.copyOf(controller.serviceRegistry().configServer().getContentClusters(new DeploymentId(id, zone)))); - return clusters.build(); - } - /** Returns the generated services.xml content for the tester application. */ static byte[] servicesXml(AthenzDomain domain, boolean useAthenzCredentials, boolean useTesterCertificate, NodeResources resources) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 0ecce359a02..54b5c339159 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.net.URI; import java.security.cert.X509Certificate; @@ -351,12 +352,22 @@ public class JobController { /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */ public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage) { + if ( ! type.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); + + if ( controller.tenants().require(id.tenant()).type() == Tenant.Type.user + && controller.applications().getApplication(TenantAndApplicationId.from(id)).isEmpty()) + controller.applications().createApplication(TenantAndApplicationId.from(id), Optional.empty()); + controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().internal()) - controller.applications().store(registered(application)); + application = registered(application); + + if ( ! application.get().instances().containsKey(id.instance())) + application = application.withNewInstance(id.instance()); + + controller.applications().store(application); }); - if ( ! type.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); last(id, type).filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); locked(id, type, __ -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 4e13a1c25e5..1743cad32e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -67,7 +67,7 @@ public class ControllerMaintenance extends AbstractComponent { deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, controller.serviceRegistry().ownershipIssues()); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); - jobRunner = new JobRunner(controller, Duration.ofMinutes(2), jobControl); + jobRunner = new JobRunner(controller, Duration.ofSeconds(90), jobControl); osUpgraders = osUpgraders(controller, jobControl); osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 79ababd20d3..9253e249765 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -1,7 +1,6 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; @@ -13,17 +12,19 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; /** + * This calculates and reports system-wide metrics based on data from a {@link Controller}. + * * @author mortent * @author mpolden */ @@ -34,9 +35,12 @@ public class MetricsReporter extends Maintainer { public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds"; public static final String DEPLOYMENT_WARNINGS = "deployment.warnings"; + public static final String NODES_FAILING_SYSTEM_UPGRADE = "deployment.nodesFailingSystemUpgrade"; public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; + private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofHours(1); + private final Metric metric; private final Clock clock; @@ -51,12 +55,13 @@ public class MetricsReporter extends Maintainer { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); + reportNodesFailingSystemUpgrade(); } private void reportRemainingRotations() { try (RotationLock lock = controller().applications().rotationRepository().lock()) { int availableRotations = controller().applications().rotationRepository().availableRotations(lock).size(); - metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Collections.emptyMap())); + metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Map.of())); } } @@ -66,7 +71,7 @@ public class MetricsReporter extends Maintainer { .flatMap(application -> application.instances().values().stream()) .collect(Collectors.toUnmodifiableList()); - metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(instances) * 100, metric.createContext(Collections.emptyMap())); + metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(instances) * 100, metric.createContext(Map.of())); averageDeploymentDurations(instances, clock.instant()).forEach((application, duration) -> { metric.set(DEPLOYMENT_AVERAGE_DURATION, duration.getSeconds(), metric.createContext(dimensions(application))); @@ -93,6 +98,24 @@ public class MetricsReporter extends Maintainer { metric.set(NAME_SERVICE_REQUESTS_QUEUED, controller().curator().readNameServiceQueue().requests().size(), metric.createContext(Map.of())); } + + private void reportNodesFailingSystemUpgrade() { + metric.set(NODES_FAILING_SYSTEM_UPGRADE, nodesFailingSystemUpgrade(), metric.createContext(Map.of())); + } + + private int nodesFailingSystemUpgrade() { + if (!controller().versionStatus().isUpgrading()) return 0; + var nodesFailingUpgrade = 0; + var acceptableInstant = clock.instant().minus(NODE_UPGRADE_TIMEOUT); + for (var vespaVersion : controller().versionStatus().versions()) { + if (vespaVersion.confidence() == VespaVersion.Confidence.broken) continue; + for (var nodeVersion : vespaVersion.nodeVersions().asMap().values()) { + if (!nodeVersion.changing()) continue; + if (nodeVersion.changedAt().isBefore(acceptableInstant)) nodesFailingUpgrade++; + } + } + return nodesFailingUpgrade; + } private static double deploymentFailRatio(List<Instance> instances) { return instances.stream() @@ -149,10 +172,8 @@ public class MetricsReporter extends Maintainer { } private static Map<String, String> dimensions(ApplicationId application) { - return ImmutableMap.of( - "tenant", application.tenant().value(), - "app",application.application().value() + "." + application.instance().value() - ); + return Map.of("tenant", application.tenant().value(), + "app",application.application().value() + "." + application.instance().value()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 08b3355587f..61fd0b67ec9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -1,14 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableBiMap; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -21,7 +20,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -39,7 +37,7 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationState; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; -import java.security.Principal; +import java.security.PublicKey; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -54,7 +52,6 @@ import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Serializes {@link Application}s to/from slime. @@ -192,7 +189,7 @@ public class ApplicationSerializer { application.majorVersion().ifPresent(majorVersion -> root.setLong(majorVersionField, majorVersion)); root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); - deployKeysToSlime(application.pemDeployKeys().stream(), root.setArray(pemDeployKeysField)); + deployKeysToSlime(application.deployKeys(), root.setArray(pemDeployKeysField)); instancesToSlime(application, root.setArray(instancesField)); return slime; } @@ -208,8 +205,8 @@ public class ApplicationSerializer { } } - private void deployKeysToSlime(Stream<String> pemDeployKeys, Cursor array) { - pemDeployKeys.forEach(array::addString); + private void deployKeysToSlime(Set<PublicKey> deployKeys, Cursor array) { + deployKeys.forEach(key -> array.addString(KeyUtils.toPem(key))); } private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) { @@ -384,14 +381,14 @@ public class ApplicationSerializer { OptionalInt majorVersion = Serializers.optionalInteger(root.field(majorVersionField)); ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); - Set<String> pemDeployKeys = pemDeployKeysFromSlime(root.field(pemDeployKeysField)); + Set<PublicKey> deployKeys = deployKeysFromSlime(root.field(pemDeployKeysField)); List<Instance> instances = instancesFromSlime(id, deploymentSpec, root.field(instancesField)); OptionalLong projectId = Serializers.optionalLong(root.field(projectIdField)); boolean builtInternally = root.field(builtInternallyField).asBool(); return new Application(id, createdAt, deploymentSpec, validationOverrides, deploying, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKeys, projectId, builtInternally, instances); + deployKeys, projectId, builtInternally, instances); } private List<Instance> instancesFromSlime(TenantAndApplicationId id, DeploymentSpec deploymentSpec, Inspector field) { @@ -411,9 +408,9 @@ public class ApplicationSerializer { return instances; } - private Set<String> pemDeployKeysFromSlime(Inspector array) { - Set<String> keys = new LinkedHashSet<>(); - array.traverse((ArrayTraverser) (__, key) -> keys.add(key.asString())); + private Set<PublicKey> deployKeysFromSlime(Inspector array) { + Set<PublicKey> keys = new LinkedHashSet<>(); + array.traverse((ArrayTraverser) (__, key) -> keys.add(KeyUtils.fromPemEncodedPublicKey(key.asString()))); return keys; } @@ -428,7 +425,7 @@ public class ApplicationSerializer { applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)), Version.fromString(deploymentObject.field(versionField).asString()), Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), - clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)), + Map.of(), clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), DeploymentActivity.create(Serializers.optionalInstant(deploymentObject.field(lastQueriedField)), @@ -484,21 +481,6 @@ public class ApplicationSerializer { return map; } - private Map<ClusterSpec.Id, ClusterUtilization> clusterUtilsMapFromSlime(Inspector object) { - Map<ClusterSpec.Id, ClusterUtilization> map = new HashMap<>(); - object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), clusterUtililzationFromSlime(value))); - return map; - } - - private ClusterUtilization clusterUtililzationFromSlime(Inspector object) { - double cpu = object.field(clusterUtilsCpuField).asDouble(); - double mem = object.field(clusterUtilsMemField).asDouble(); - double disk = object.field(clusterUtilsDiskField).asDouble(); - double diskBusy = object.field(clusterUtilsDiskBusyField).asDouble(); - - return new ClusterUtilization(mem, cpu, disk, diskBusy); - } - private ClusterInfo clusterInfoFromSlime(Inspector inspector) { String flavor = inspector.field(clusterInfoFlavorField).asString(); int cost = (int)inspector.field(clusterInfoCostField).asLong(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 9501ac5a7f9..357dbb37b27 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -360,22 +360,11 @@ public class CuratorDb { private Stream<TenantAndApplicationId> readApplicationIds() { return curator.getChildren(applicationRoot).stream() - .filter(id -> id.split(":").length == 2) .map(TenantAndApplicationId::fromSerialized); } - public void deleteOldApplicationData() { - curator.getChildren(applicationRoot).stream() - .filter(id -> id.split(":").length == 3) - .forEach(id -> curator.delete(applicationRoot.append(id))); - } - - // TODO jonmv: Refactor when instance split operation is done - public void storeWithoutInstance(Application application) { - if (application.instances().isEmpty()) - curator.delete(applicationPath(application.id())); - else - writeApplication(application); + public void removeApplication(TenantAndApplicationId id) { + curator.delete(applicationPath(id)); } // -------------- Job Runs ------------------------------------------------ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 78d166607df..35128466e4d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; +import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -22,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import java.net.URI; import java.security.Principal; +import java.security.PublicKey; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -91,14 +93,14 @@ public class TenantSerializer { } private void toSlime(CloudTenant tenant, Cursor root) { - pemDeveloperKeysToSlime(tenant.pemDeveloperKeys(), root.setArray(pemDeveloperKeysField)); + developerKeysToSlime(tenant.developerKeys(), root.setArray(pemDeveloperKeysField)); toSlime(tenant.billingInfo(), root.setObject(billingInfoField)); } - private void pemDeveloperKeysToSlime(BiMap<String, Principal> keys, Cursor array) { + private void developerKeysToSlime(BiMap<PublicKey, Principal> keys, Cursor array) { keys.forEach((key, user) -> { Cursor object = array.addObject(); - object.setString("key", key); + object.setString("key", KeyUtils.toPem(key)); object.setString("user", user.getName()); }); } @@ -139,15 +141,16 @@ public class TenantSerializer { private CloudTenant cloudTenantFrom(Inspector tenantObject) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); BillingInfo billingInfo = billingInfoFrom(tenantObject.field(billingInfoField)); - BiMap<String, Principal> pemDeveloperKeys = pemDeveloperKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); - return new CloudTenant(name, billingInfo, pemDeveloperKeys); + BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); + return new CloudTenant(name, billingInfo, developerKeys); } - private BiMap<String, Principal> pemDeveloperKeysFromSlime(Inspector array) { - ImmutableBiMap.Builder<String, Principal> keys = ImmutableBiMap.builder(); - array.traverse((ArrayTraverser) (__, keyObject) -> { - keys.put(keyObject.field("key").asString(), new SimplePrincipal(keyObject.field("user").asString())); - }); + private BiMap<PublicKey, Principal> developerKeysFromSlime(Inspector array) { + ImmutableBiMap.Builder<PublicKey, Principal> keys = ImmutableBiMap.builder(); + array.traverse((ArrayTraverser) (__, keyObject) -> + keys.put(KeyUtils.fromPemEncodedPublicKey(keyObject.field("key").asString()), + new SimplePrincipal(keyObject.field("user").asString()))); + return keys.build(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 207a5f8dcf9..5061f32da68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -1,6 +1,7 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -9,6 +10,8 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -47,6 +50,14 @@ public class VersionStatusSerializer { private static final String confidenceField = "confidence"; private static final String configServersField = "configServerHostnames"; + // NodeVersions fields + private static final String nodeVersionsField = "nodeVersions"; + + // NodeVersion fields + private static final String hostnameField = "hostname"; + private static final String wantedVersionField = "wantedVersion"; + private static final String changedAtField = "changedAt"; + // DeploymentStatistics fields private static final String versionField = "version"; private static final String failingField = "failing"; @@ -77,9 +88,20 @@ public class VersionStatusSerializer { object.setBool(isReleasedField, version.isReleased()); deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField)); object.setString(confidenceField, version.confidence().name()); - configServersToSlime(version.systemApplicationHostnames(), object.setArray(configServersField)); + configServersToSlime(version.nodeVersions().hostnames(), object.setArray(configServersField)); + nodeVersionsToSlime(version.nodeVersions(), object.setArray(nodeVersionsField)); + } + + private void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) { + for (NodeVersion nodeVersion : nodeVersions.asMap().values()) { + var nodeVersionObject = array.addObject(); + nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value()); + nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString()); + nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli()); + } } + // TODO(mpolden): Remove after October 2019 private void configServersToSlime(Set<HostName> configServerHostnames, Cursor array) { configServerHostnames.stream().map(HostName::value).forEach(array::addString); } @@ -102,17 +124,38 @@ public class VersionStatusSerializer { } private VespaVersion vespaVersionFromSlime(Inspector object) { - return new VespaVersion(deploymentStatisticsFromSlime(object.field(deploymentStatisticsField)), + var deploymentStatistics = deploymentStatisticsFromSlime(object.field(deploymentStatisticsField)); + return new VespaVersion(deploymentStatistics, object.field(releaseCommitField).asString(), Instant.ofEpochMilli(object.field(committedAtField).asLong()), object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), - object.field(isReleasedField).valid() ? object.field(isReleasedField).asBool() : true, - configServersFromSlime(object.field(configServersField)), + object.field(isReleasedField).asBool(), + nodeVersionsFromSlime(object, deploymentStatistics.version()), VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) ); } + private NodeVersions nodeVersionsFromSlime(Inspector root, Version version) { + var nodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); + var nodeVersionsRoot = root.field(nodeVersionsField); + if (nodeVersionsRoot.valid()) { + nodeVersionsRoot.traverse((ArrayTraverser) (i, entry) -> { + var hostname = HostName.from(entry.field(hostnameField).asString()); + var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); + var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong()); + nodeVersions.put(hostname, new NodeVersion(hostname, version, wantedVersion, changedAt)); + }); + } else { + // TODO(mpolden): Remove after October 2019 + var configServerHostnames = configServersFromSlime(root.field(configServersField)); + for (var hostname : configServerHostnames) { + nodeVersions.put(hostname, NodeVersion.empty(hostname)); + } + } + return new NodeVersions(nodeVersions.build()); + } + private Set<HostName> configServersFromSlime(Inspector array) { Set<HostName> configServerHostnames = new LinkedHashSet<>(); array.traverse((ArrayTraverser) (i, entry) -> configServerHostnames.add(HostName.from(entry.asString()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 4c4478c9af6..24819fda261 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -6,8 +6,8 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -16,19 +16,23 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; +import com.yahoo.security.KeyUtils; +import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.AlreadyExistsException; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.ActivateResult; @@ -69,10 +73,6 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.ResourceResponse; -import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationState; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; @@ -96,12 +96,12 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.DigestInputStream; import java.security.Principal; +import java.security.PublicKey; import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Base64; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -132,6 +132,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private final Controller controller; private final AccessControlRequests accessControlRequests; + private final TestConfigSerializer testConfigSerializer; @Inject public ApplicationApiHandler(LoggingRequestHandler.Context parentCtx, @@ -140,6 +141,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { super(parentCtx); this.controller = controller; this.accessControlRequests = accessControlRequests; + this.testConfigSerializer = new TestConfigSerializer(controller.system()); } @Override @@ -240,14 +242,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/key")) return addDeveloperKey(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return addDeployKey(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), "default", request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); @@ -377,9 +379,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Principal user = request.getJDiscRequest().getUserPrincipal(); String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString(); - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> - controller.tenants().store(tenant.withPemDeveloperKey(pemDeveloperKey, user))); - return new MessageResponse("Set developer key " + pemDeveloperKey + " for " + user); + PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey); + Slime root = new Slime(); + controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> { + tenant = tenant.withDeveloperKey(developerKey, user); + toSlime(root.setObject().setArray("keys"), tenant.get().developerKeys()); + controller.tenants().store(tenant); + }); + return new SlimeJsonResponse(root); } private HttpResponse removeDeveloperKey(String tenantName, HttpRequest request) { @@ -387,26 +394,51 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString(); - Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).pemDeveloperKeys().get(pemDeveloperKey); - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> - controller.tenants().store(tenant.withoutPemDeveloperKey(pemDeveloperKey))); - return new MessageResponse("Removed developer key " + pemDeveloperKey + " for " + user); + PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey); + Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).developerKeys().get(developerKey); + Slime root = new Slime(); + controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> { + tenant = tenant.withoutDeveloperKey(developerKey); + toSlime(root.setObject().setArray("keys"), tenant.get().developerKeys()); + controller.tenants().store(tenant); + }); + return new SlimeJsonResponse(root); + } + + private void toSlime(Cursor keysArray, Map<PublicKey, Principal> keys) { + keys.forEach((key, principal) -> { + Cursor keyObject = keysArray.addObject(); + keyObject.setString("key", KeyUtils.toPem(key)); + keyObject.setString("user", principal.getName()); + }); } private HttpResponse addDeployKey(String tenantName, String applicationName, HttpRequest request) { String pemDeployKey = toSlime(request.getData()).get().field("key").asString(); + PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey); + Slime root = new Slime(); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> { - controller.applications().store(application.withPemDeployKey(pemDeployKey)); + application = application.withDeployKey(deployKey); + application.get().deployKeys().stream() + .map(KeyUtils::toPem) + .forEach(root.setObject().setArray("keys")::addString); + controller.applications().store(application); }); - return new MessageResponse("Added deploy key " + pemDeployKey); + return new SlimeJsonResponse(root); } private HttpResponse removeDeployKey(String tenantName, String applicationName, HttpRequest request) { String pemDeployKey = toSlime(request.getData()).get().field("key").asString(); + PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey); + Slime root = new Slime(); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> { - controller.applications().store(application.withoutPemDeployKey(pemDeployKey)); + application = application.withoutDeployKey(deployKey); + application.get().deployKeys().stream() + .map(KeyUtils::toPem) + .forEach(root.setObject().setArray("keys")::addString); + controller.applications().store(application); }); - return new MessageResponse("Removed deploy key " + pemDeployKey); + return new SlimeJsonResponse(root); } private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { @@ -424,7 +456,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector pemDeployKeyField = requestObject.field("pemDeployKey"); if (pemDeployKeyField.valid()) { String pemDeployKey = pemDeployKeyField.asString(); - application = application.withPemDeployKey(pemDeployKey); + PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey); + application = application.withDeployKey(deployKey); messageBuilder.add("Added deploy key " + pemDeployKey); } @@ -654,9 +687,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } // TODO jonmv: Remove when clients are updated - application.pemDeployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", key)); + application.deployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", KeyUtils.toPem(key))); - application.pemDeployKeys().forEach(object.setArray("pemDeployKeys")::addString); + application.deployKeys().stream().map(KeyUtils::toPem).forEach(object.setArray("pemDeployKeys")::addString); // Metrics Cursor metricsObject = object.setObject("metrics"); @@ -1017,25 +1050,30 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return tenant(controller.tenants().require(TenantName.from(tenantName)), request); } - private HttpResponse createApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { + private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); - try { - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest())); - Application application = controller.applications().createApplication(id, credentials); - - Slime slime = new Slime(); - toSlime(id, slime.setObject(), request); - return new SlimeJsonResponse(slime); - } - catch (ZmsClientException e) { // TODO: Push conversion down - if (e.getErrorCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) - throw new ForbiddenException("Not authorized to create application", e); - else - throw e; - } + TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); + Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user + ? Optional.empty() + : Optional.of(accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest())); + Application application = controller.applications().createApplication(id, credentials); + + Slime slime = new Slime(); + toSlime(id, slime.setObject(), request); + return new SlimeJsonResponse(slime); + } + + // TODO jonmv: Remove when clients are updated. + private HttpResponse createInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) { + TenantAndApplicationId applicationId = TenantAndApplicationId.from(tenantName, applicationName); + if (controller.applications().getApplication(applicationId).isEmpty()) + createApplication(tenantName, applicationName, request); + + controller.applications().createInstance(applicationId.instance(instanceName)); + + Slime slime = new Slime(); + toSlime(applicationId.instance(instanceName), slime.setObject(), request); + return new SlimeJsonResponse(slime); } /** Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9". */ @@ -1227,12 +1265,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deployOptions.field("ignoreValidationErrors").asBool(), deployOptions.field("deployCurrentVersion").asBool()); + applicationPackage.ifPresent(aPackage -> controller.applications().verifyApplicationIdentityConfiguration(applicationId.tenant(), + aPackage, + Optional.of(requireUserPrincipal(request)))); + ActivateResult result = controller.applications().deploy(applicationId, zone, applicationPackage, applicationVersion, - deployOptionsJsonClass, - Optional.of(requireUserPrincipal(request))); + deployOptionsJsonClass); return new SlimeJsonResponse(toSlime(result)); } @@ -1255,22 +1296,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { - TenantName tenant = TenantName.from(tenantName); - ApplicationName application = ApplicationName.from(applicationName); - Optional<Credentials> credentials = controller.tenants().require(tenant).type() == Tenant.Type.user + TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); + Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user ? Optional.empty() - : Optional.of(accessControlRequests.credentials(tenant, toSlime(request.getData()).get(), request.getJDiscRequest())); - controller.applications().deleteApplication(tenant, application, credentials); - return new MessageResponse("Deleted application " + tenant + "." + application); + : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); + controller.applications().deleteApplication(id, credentials); + return new MessageResponse("Deleted application " + id); } private HttpResponse deleteInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); + TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user ? Optional.empty() : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); - controller.applications().deleteInstance(id, credentials); - return new MessageResponse("Deleted instance " + id.toFullString()); + controller.applications().deleteInstance(id.instance(instanceName)); + if (controller.applications().requireApplication(id).instances().isEmpty()) + controller.applications().deleteApplication(id, credentials); + return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString()); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { @@ -1300,11 +1342,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse testConfig(ApplicationId id, JobType type) { - var endpoints = controller.applications().clusterEndpoints(id, controller.jobController().testedZoneAndProductionZones(id, type)); - return new SlimeJsonResponse(new TestConfigSerializer(controller.system()).configSlime(id, - type, - endpoints, - Collections.emptyMap())); + Set<ZoneId> zones = controller.jobController().testedZoneAndProductionZones(id, type); + return new SlimeJsonResponse(testConfigSerializer.configSlime(id, + type, + controller.applications().clusterEndpoints(id, zones), + controller.applications().contentClustersByZone(id, zones))); } private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) { @@ -1366,18 +1408,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { case cloud: { CloudTenant cloudTenant = (CloudTenant) tenant; - Cursor pemDeployKeysArray = object.setArray("pemDeployKeys"); - for (Application application : applications) - for (String key : application.pemDeployKeys()) { - Cursor keyObject = pemDeployKeysArray.addObject(); - keyObject.setString("key", key); - keyObject.setString("application", application.id().application().value()); - } - Cursor pemDeveloperKeysArray = object.setArray("pemDeveloperKeys"); - cloudTenant.pemDeveloperKeys().forEach((key, user) -> { + cloudTenant.developerKeys().forEach((key, user) -> { Cursor keyObject = pemDeveloperKeysArray.addObject(); - keyObject.setString("key", key); + keyObject.setString("key", KeyUtils.toPem(key)); keyObject.setString("user", user.getName()); }); @@ -1470,6 +1504,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return Joiner.on("/").join(elements); } + private void toSlime(TenantAndApplicationId id, Cursor object, HttpRequest request) { + object.setString("tenant", id.tenant().value()); + object.setString("application", id.application().value()); + object.setString("url", withPath("/application/v4" + + "/tenant/" + id.tenant().value() + + "/application/" + id.application().value(), + request.getUri()).toString()); + } + private void toSlime(ApplicationId id, Cursor object, HttpRequest request) { object.setString("tenant", id.tenant().value()); object.setString("application", id.application().value()); @@ -1639,6 +1682,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { long projectId = Math.max(1, submitOptions.field("projectId").asLong()); ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); + if (DeploymentSpec.empty.equals(applicationPackage.deploymentSpec())) + throw new IllegalArgumentException("Missing required file 'deployment.xml'"); + controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), applicationPackage, Optional.of(requireUserPrincipal(request))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index ab92e38ee4b..49015f16cce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -64,6 +64,9 @@ import static java.util.stream.Collectors.toMap; * * @see JobController * @see ApplicationApiHandler + * + * @author smorgrav + * @author jonmv */ class JobControllerApiHandlerHelper { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 86310ca2f6b..2adf6ce95e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.component.Version; @@ -91,7 +91,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { versionObject.setBool("systemVersion", version.isSystemVersion()); Cursor configServerArray = versionObject.setArray("configServers"); - for (HostName hostname : version.systemApplicationHostnames()) { + for (HostName hostname : version.nodeVersions().hostnames()) { Cursor configServerObject = configServerArray.addObject(); configServerObject.setString("hostname", hostname.value()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 6755110bb49..7ad2e03ef1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -10,19 +10,27 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; import com.yahoo.log.LogLevel; +import com.yahoo.security.KeyUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.yolean.Exceptions; import java.security.Principal; +import java.security.PublicKey; +import java.util.Base64; +import java.util.HashSet; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.logging.Logger; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Assigns the {@link Role#buildService(TenantName, ApplicationName)} role to requests with a * Authorization header signature matching the public key of the indicated application. @@ -46,25 +54,11 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase { if ( request.getAttribute(SecurityContext.ATTRIBUTE_NAME) == null && request.getHeader("X-Authorization") != null) try { - ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id")); - boolean verified = controller.applications().getApplication(TenantAndApplicationId.from(id)).stream() - .flatMap(application -> application.pemDeployKeys().stream()) - .map(key -> new RequestVerifier(key, controller.clock())) - .anyMatch(verifier -> verifier.verify(Method.valueOf(request.getMethod()), - request.getUri(), - request.getHeader("X-Timestamp"), - request.getHeader("X-Content-Hash"), - request.getHeader("X-Authorization"))); - - if (verified) { - Principal principal = new SimplePrincipal("buildService@" + id.tenant() + "." + id.application()); - request.setUserPrincipal(principal); - request.setRemoteUser(principal.getName()); - request.setAttribute(SecurityContext.ATTRIBUTE_NAME, - new SecurityContext(principal, - Set.of(Role.buildService(id.tenant(), id.application()), - Role.applicationDeveloper(id.tenant(), id.application())))); - } + getSecurityContext(request).ifPresent(securityContext -> { + request.setUserPrincipal(securityContext.principal()); + request.setRemoteUser(securityContext.principal().getName()); + request.setAttribute(SecurityContext.ATTRIBUTE_NAME, securityContext); + }); } catch (Exception e) { logger.log(LogLevel.DEBUG, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e)); @@ -72,4 +66,48 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase { return Optional.empty(); } + // TODO jonmv: Remove after October 2019. + private boolean anyDeployKeyMatches(TenantAndApplicationId id, DiscFilterRequest request) { + return controller.applications().getApplication(id).stream() + .map(Application::deployKeys) + .flatMap(Set::stream) + .anyMatch(key -> keyVerifies(key, request)); + } + + private boolean keyVerifies(PublicKey key, DiscFilterRequest request) { + return new RequestVerifier(key, controller.clock()).verify(Method.valueOf(request.getMethod()), + request.getUri(), + request.getHeader("X-Timestamp"), + request.getHeader("X-Content-Hash"), + request.getHeader("X-Authorization")); + } + + private Optional<SecurityContext> getSecurityContext(DiscFilterRequest request) { + ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id")); + if (request.getHeader("X-Key") != null) { // TODO jonmv: Remove check and else branch after Oct 2019. + PublicKey key = KeyUtils.fromPemEncodedPublicKey(new String(Base64.getDecoder().decode(request.getHeader("X-Key")), UTF_8)); + if (keyVerifies(key, request)) { + Optional<CloudTenant> tenant = controller.tenants().get(id.tenant()) + .filter(CloudTenant.class::isInstance) + .map(CloudTenant.class::cast); + if (tenant.isPresent() && tenant.get().developerKeys().containsKey(key)) + return Optional.of(new SecurityContext(tenant.get().developerKeys().get(key), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); + + Optional <Application> application = controller.applications().getApplication(TenantAndApplicationId.from(id)); + if (application.isPresent() && application.get().deployKeys().contains(key)) + return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); // TODO jonmv: Change to headless after Oct 10 2019. + } + } + else if (anyDeployKeyMatches(TenantAndApplicationId.from(id), request)) + return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); + + return Optional.empty(); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 807e74b7c75..77622df4c4a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -26,10 +26,9 @@ import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.yolean.Exceptions; -import java.security.Principal; +import java.security.PublicKey; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -200,9 +199,9 @@ public class UserApiHandler extends LoggingRequestHandler { // TODO jonmv: Change to developer role, when this exists. if (role.definition().equals(RoleDefinition.tenantOperator)) controller.tenants().lockIfPresent(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> { - String key = tenant.get().pemDeveloperKeys().inverse().get(new SimplePrincipal(user.value())); + PublicKey key = tenant.get().developerKeys().inverse().get(new SimplePrincipal(user.value())); if (key != null) - controller.tenants().store(tenant.withoutPemDeveloperKey(key)); + controller.tenants().store(tenant.withoutDeveloperKey(key)); }); users.removeUsers(role, List.of(user)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java index 77ccce873fe..66c87a8eefd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.util.List; @@ -52,7 +53,7 @@ public interface AccessControl { * @param id the ID of the application to create * @param credentials the credentials for the entity requesting the creation */ - void createApplication(ApplicationId id, Credentials credentials); + void createApplication(TenantAndApplicationId id, Credentials credentials); /** * Deletes access control for the given tenant. @@ -60,7 +61,7 @@ public interface AccessControl { * @param id the ID of the application to delete * @param credentials the credentials for the entity requesting the deletion */ - void deleteApplication(ApplicationId id, Credentials credentials); + void deleteApplication(TenantAndApplicationId id, Credentials credentials); /** * Returns the list of tenants to which a user has access. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index 7da3e43c9a5..a88e38e5f89 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -1,10 +1,8 @@ package com.yahoo.vespa.hosted.controller.security; import com.google.inject.Inject; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; @@ -12,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.TenantRole; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -58,14 +57,14 @@ public class CloudAccessControl implements AccessControl { } @Override - public void createApplication(ApplicationId id, Credentials credentials) { + public void createApplication(TenantAndApplicationId id, Credentials credentials) { for (Role role : Roles.applicationRoles(id.tenant(), id.application())) userManagement.createRole(role); userManagement.addUsers(Role.applicationAdmin(id.tenant(), id.application()), List.of(new UserId(credentials.user().getName()))); } @Override - public void deleteApplication(ApplicationId id, Credentials credentials) { + public void deleteApplication(TenantAndApplicationId id, Credentials credentials) { for (ApplicationRole role : Roles.applicationRoles(id.tenant(), id.application())) userManagement.deleteRole(role); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 6ef9b5e6a4f..e230daf0c50 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import java.security.Principal; +import java.security.PublicKey; import java.util.Objects; import java.util.Optional; @@ -17,13 +18,13 @@ import java.util.Optional; public class CloudTenant extends Tenant { private final BillingInfo billingInfo; - private final BiMap<String, Principal> pemDeveloperKeys; + private final BiMap<PublicKey, Principal> developerKeys; /** Public for the serialization layer — do not use! */ - public CloudTenant(TenantName name, BillingInfo info, BiMap<String, Principal> pemDeveloperKeys) { + public CloudTenant(TenantName name, BillingInfo info, BiMap<PublicKey, Principal> developerKeys) { super(name, Optional.empty()); billingInfo = info; - this.pemDeveloperKeys = pemDeveloperKeys; + this.developerKeys = developerKeys; } /** Creates a tenant with the given name, provided it passes validation. */ @@ -37,7 +38,7 @@ public class CloudTenant extends Tenant { public BillingInfo billingInfo() { return billingInfo; } /** Returns the set of developer keys and their corresponding developers for this tenant. */ - public BiMap<String, Principal> pemDeveloperKeys() { return pemDeveloperKeys; } + public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; } @Override public Type type() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java new file mode 100644 index 00000000000..0a690b90410 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java @@ -0,0 +1,93 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.HostName; + +import java.time.Instant; +import java.util.Objects; + +/** + * Version information for a node allocated to a {@link com.yahoo.vespa.hosted.controller.application.SystemApplication}. + * + * This is immutable. + * + * @author mpolden + */ +public class NodeVersion { + + private final HostName hostname; + private final Version currentVersion; + private final Version wantedVersion; + private final Instant changedAt; + + public NodeVersion(HostName hostname, Version currentVersion, Version wantedVersion, Instant changedAt) { + this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.currentVersion = Objects.requireNonNull(currentVersion, "version must be non-null"); + this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); + this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); + } + + /** Hostname of this */ + public HostName hostname() { + return hostname; + } + + /** Current version of this */ + public Version currentVersion() { + return currentVersion; + } + + /** Wanted version of this */ + public Version wantedVersion() { + return wantedVersion; + } + + /** Returns whether this is changing (upgrading or downgrading) */ + public boolean changing() { + return !currentVersion.equals(wantedVersion); + } + + /** The most recent time the version of this changed */ + public Instant changedAt() { + return changedAt; + } + + /** Returns a copy of this with current version set to given version */ + public NodeVersion withCurrentVersion(Version version, Instant changedAt) { + if (currentVersion.equals(version)) return this; + return new NodeVersion(hostname, version, wantedVersion, changedAt); + } + + /** Returns a copy of this with wanted version set to given version */ + public NodeVersion withWantedVersion(Version version) { + if (wantedVersion.equals(version)) return this; + return new NodeVersion(hostname, currentVersion, version, changedAt); + } + + @Override + public String toString() { + return hostname + ": " + currentVersion + " -> " + wantedVersion + " [changedAt=" + changedAt + "]"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeVersion that = (NodeVersion) o; + return hostname.equals(that.hostname) && + currentVersion.equals(that.currentVersion) && + wantedVersion.equals(that.wantedVersion) && + changedAt.equals(that.changedAt); + } + + @Override + public int hashCode() { + return Objects.hash(hostname, currentVersion, wantedVersion, changedAt); + } + + public static NodeVersion empty(HostName hostname) { + return new NodeVersion(hostname, Version.emptyVersion, Version.emptyVersion, Instant.EPOCH); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java new file mode 100644 index 00000000000..3ab96e03bcd --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java @@ -0,0 +1,97 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.versions; + +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.HostName; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; + +/** + * A filterable list of {@link NodeVersion}s. This is immutable. + * + * @author mpolden + */ +public class NodeVersions { + + public static final NodeVersions EMPTY = new NodeVersions(ImmutableMap.of()); + + private final ImmutableMap<HostName, NodeVersion> nodeVersions; + + public NodeVersions(ImmutableMap<HostName, NodeVersion> nodeVersions) { + this.nodeVersions = Objects.requireNonNull(nodeVersions); + } + + public Map<HostName, NodeVersion> asMap() { + return nodeVersions; + } + + /** Returns host names in this, grouped by version */ + public ListMultimap<Version, HostName> asVersionMap() { + var versions = ImmutableListMultimap.<Version, HostName>builder(); + for (var kv : nodeVersions.entrySet()) { + versions.put(kv.getValue().currentVersion(), kv.getKey()); + } + return versions.build(); + } + + /** Returns host names in this */ + public Set<HostName> hostnames() { + return nodeVersions.keySet(); + } + + /** Returns a copy of this containing only node versions of given version */ + public NodeVersions matching(Version version) { + return filter(nodeVersion -> nodeVersion.currentVersion().equals(version)); + } + + /** Returns number of node versions in this */ + public int size() { + return nodeVersions.size(); + } + + /** Returns a copy of this containing only the given node versions */ + public NodeVersions with(List<NodeVersion> nodeVersions) { + var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); + for (var nodeVersion : nodeVersions) { + var existing = this.nodeVersions.get(nodeVersion.hostname()); + if (existing != null) { + newNodeVersions.put(nodeVersion.hostname(), existing.withCurrentVersion(nodeVersion.currentVersion(), + nodeVersion.changedAt()) + .withWantedVersion(nodeVersion.wantedVersion())); + } else { + newNodeVersions.put(nodeVersion.hostname(), nodeVersion); + } + } + return new NodeVersions(newNodeVersions.build()); + } + + private NodeVersions filter(Predicate<NodeVersion> predicate) { + var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); + for (var kv : nodeVersions.entrySet()) { + if (!predicate.test(kv.getValue())) continue; + newNodeVersions.put(kv.getKey(), kv.getValue()); + } + return new NodeVersions(newNodeVersions.build()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeVersions that = (NodeVersions) o; + return nodeVersions.equals(that.nodeVersions); + } + + @Override + public int hashCode() { + return Objects.hash(nodeVersions); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 9dc6b86e4be..bb43ec20234 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -6,12 +6,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ListMultimap; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobList; @@ -70,7 +68,8 @@ public class VersionStatus { /** Returns whether the system is currently upgrading */ public boolean isUpgrading() { return systemVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion) - .isBefore(controllerVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion)); + .isBefore(controllerVersion().map(VespaVersion::versionNumber) + .orElse(Version.emptyVersion)); } /** @@ -91,14 +90,14 @@ public class VersionStatus { /** Create a full, updated version status. This is expensive and should be done infrequently */ public static VersionStatus compute(Controller controller) { - ListMultimap<Version, HostName> systemApplicationVersions = findSystemApplicationVersions(controller); - ListMultimap<ControllerVersion, HostName> controllerVersions = findControllerVersions(controller); + var systemApplicationVersions = findSystemApplicationVersions(controller); + var controllerVersions = findControllerVersions(controller); - ListMultimap<Version, HostName> infrastructureVersions = ArrayListMultimap.create(); + var infrastructureVersions = ArrayListMultimap.<Version, HostName>create(); for (var kv : controllerVersions.asMap().entrySet()) { infrastructureVersions.putAll(kv.getKey().version(), kv.getValue()); } - infrastructureVersions.putAll(systemApplicationVersions); + infrastructureVersions.putAll(systemApplicationVersions.asVersionMap()); // The controller version is the lowest controller version of all controllers ControllerVersion controllerVersion = controllerVersions.keySet().stream() @@ -138,7 +137,7 @@ public class VersionStatus { controllerVersion, systemVersion, isReleased, - systemApplicationVersions.get(statistics.version()), + systemApplicationVersions.matching(statistics.version()), controller); versions.add(vespaVersion); } catch (IllegalArgumentException e) { @@ -152,29 +151,32 @@ public class VersionStatus { return new VersionStatus(versions); } - private static ListMultimap<Version, HostName> findSystemApplicationVersions(Controller controller) { - ListMultimap<Version, HostName> versions = ArrayListMultimap.create(); - for (ZoneApi zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { - for (SystemApplication application : SystemApplication.all()) { - List<Node> eligibleForUpgradeApplicationNodes = controller.serviceRegistry().configServer().nodeRepository() - .list(zone.getId(), application.id()).stream() - .filter(SystemUpgrader::eligibleForUpgrade) - .collect(Collectors.toList()); - if (eligibleForUpgradeApplicationNodes.isEmpty()) - continue; - - boolean configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty()); + private static NodeVersions findSystemApplicationVersions(Controller controller) { + var nodeVersions = controller.versionStatus().systemVersion() + .map(VespaVersion::nodeVersions) + .orElse(NodeVersions.EMPTY); + var newNodeVersions = new ArrayList<NodeVersion>(); + for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { + for (var application : SystemApplication.all()) { + var nodes = controller.serviceRegistry().configServer().nodeRepository() + .list(zone.getId(), application.id()).stream() + .filter(SystemUpgrader::eligibleForUpgrade) + .collect(Collectors.toList()); + if (nodes.isEmpty()) continue; + var configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty()); if (!configConverged) { - log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() + " has not converged"); + log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() + + " has not converged"); } - for (Node node : eligibleForUpgradeApplicationNodes) { + var now = controller.clock().instant(); + for (var node : nodes) { // Only use current node version if config has converged - Version nodeVersion = configConverged ? node.currentVersion() : controller.systemVersion(); - versions.put(nodeVersion, node.hostname()); + Version version = configConverged ? node.currentVersion() : controller.systemVersion(); + newNodeVersions.add(new NodeVersion(node.hostname(), version, node.wantedVersion(), now)); } } } - return versions; + return nodeVersions.with(newNodeVersions); } private static ListMultimap<ControllerVersion, HostName> findControllerVersions(Controller controller) { @@ -241,7 +243,7 @@ public class VersionStatus { ControllerVersion controllerVersion, Version systemVersion, boolean isReleased, - Collection<HostName> configServerHostnames, + NodeVersions nodeVersions, Controller controller) { var isSystemVersion = statistics.version().equals(systemVersion); var isControllerVersion = statistics.version().equals(controllerVersion.version()); @@ -279,7 +281,7 @@ public class VersionStatus { isControllerVersion, isSystemVersion, isReleased, - configServerHostnames, + nodeVersions, confidence); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index dc0b2c12d5c..0d144913022 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -1,16 +1,12 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import java.time.Instant; import java.time.ZoneOffset; -import java.util.Collection; -import java.util.Set; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; @@ -30,12 +26,12 @@ public class VespaVersion implements Comparable<VespaVersion> { private final boolean isSystemVersion; private final boolean isReleased; private final DeploymentStatistics statistics; - private final ImmutableSet<HostName> systemApplicationHostnames; + private final NodeVersions nodeVersions; private final Confidence confidence; public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant committedAt, boolean isControllerVersion, boolean isSystemVersion, boolean isReleased, - Collection<HostName> systemApplicationHostnames, + NodeVersions nodeVersions, Confidence confidence) { this.statistics = statistics; this.releaseCommit = releaseCommit; @@ -43,7 +39,7 @@ public class VespaVersion implements Comparable<VespaVersion> { this.isControllerVersion = isControllerVersion; this.isSystemVersion = isSystemVersion; this.isReleased = isReleased; - this.systemApplicationHostnames = ImmutableSet.copyOf(systemApplicationHostnames); + this.nodeVersions = nodeVersions; this.confidence = confidence; } @@ -108,9 +104,11 @@ public class VespaVersion implements Comparable<VespaVersion> { /** Returns whether the artifacts of this release are available in the configured maven repository. */ public boolean isReleased() { return isReleased; } - /** Returns the hosts allocated to system applications (across all zones) which are currently of this version */ - public Set<HostName> systemApplicationHostnames() { return systemApplicationHostnames; } - + /** Returns the versions of nodes allocated to system applications (across all zones) */ + public NodeVersions nodeVersions() { + return nodeVersions; + } + /** Returns the confidence we have in this versions suitability for production */ public Confidence confidence() { return confidence; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index fab1ed2ab20..e3682a78b7d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -507,7 +507,7 @@ public class ControllerTest { tester.deployAndNotify(tester.defaultInstance(app1.id()).id(), Optional.of(applicationPackage), true, systemTest); tester.applications().deactivate(app1.id().defaultInstance(), ZoneId.from(Environment.test, RegionName.from("us-east-1"))); tester.applications().deactivate(app1.id().defaultInstance(), ZoneId.from(Environment.staging, RegionName.from("us-east-3"))); - tester.applications().deleteApplication(app1.id().tenant(), app1.id().application(), tester.controllerTester().credentialsFor(app1.id())); + tester.applications().deleteApplication(app1.id(), tester.controllerTester().credentialsFor(app1.id())); try (RotationLock lock = tester.applications().rotationRepository().lock()) { assertTrue("Rotation is unassigned", tester.applications().rotationRepository().availableRotations(lock) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index cefdc3bed61..2c88d122e8f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -233,11 +233,12 @@ public final class ControllerTester { } public Application createApplication(TenantName tenant, String applicationName, String instanceName, long projectId) { - ApplicationId applicationId = ApplicationId.from(tenant.value(), applicationName, instanceName); - controller().applications().createApplication(applicationId, credentialsFor(TenantAndApplicationId.from(applicationId))); - controller().applications().lockApplicationOrThrow(TenantAndApplicationId.from(applicationId), application -> + TenantAndApplicationId applicationId = TenantAndApplicationId.from(tenant.value(), applicationName); + controller().applications().createApplication(applicationId, credentialsFor(applicationId)); + controller().applications().lockApplicationOrThrow(applicationId, application -> controller().applications().store(application.withProjectId(OptionalLong.of(projectId)))); - Application application = controller().applications().requireApplication(TenantAndApplicationId.from(applicationId)); + controller().applications().createInstance(applicationId.instance(instanceName)); + Application application = controller().applications().requireApplication(applicationId); assertTrue(application.projectId().isPresent()); return application; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 5dc6fb183a2..61b393efbff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -148,8 +148,13 @@ public class DeploymentTester { /** Upgrade system applications in all zones to given version */ public void upgradeSystemApplications(Version version) { + upgradeSystemApplications(version, SystemApplication.all()); + } + + /** Upgrade given system applications in all zones to version */ + public void upgradeSystemApplications(Version version, List<SystemApplication> systemApplications) { for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { - for (SystemApplication application : SystemApplication.all()) { + for (SystemApplication application : systemApplications) { tester.configServer().setVersion(application.id(), zone.getId(), version); tester.configServer().convergeServices(application.id(), zone.getId()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 4a7ee8bcb63..6da77a967f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.google.inject.Inject; @@ -110,7 +110,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer List<Node> nodes = IntStream.rangeClosed(1, 3) .mapToObj(i -> new Node( HostName.from("node-" + i + "-" + application.id().application() - .value()), + .value() + + "-" + zone.value()), Node.State.active, application.nodeType(), Optional.of(application.id()), initialVersion, @@ -150,9 +151,16 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Set version for an application in a given zone */ public void setVersion(ApplicationId application, ZoneId zone, Version version) { + setVersion(application, zone, version, -1); + } + + /** Set version for nodeCount number of nodes in application in a given zone */ + public void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) { + int n = 0; for (Node node : nodeRepository().list(zone, application)) { nodeRepository().putByHostname(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(), version, version)); + if (++n == nodeCount) break; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index ff245e2e488..c6bd4bde410 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -218,7 +218,7 @@ public class JobRunnerTest { // Thread is still trying to deploy tester -- delete application, and see all data is garbage collected. assertEquals(Collections.singletonList(runId), jobs.active().stream().map(run -> run.id()).collect(Collectors.toList())); - tester.controllerTester().controller().applications().deleteApplication(id.tenant(), id.application(), tester.controllerTester().credentialsFor(TenantAndApplicationId.from(id))); + tester.controllerTester().controller().applications().deleteApplication(TenantAndApplicationId.from(id), tester.controllerTester().credentialsFor(TenantAndApplicationId.from(id))); assertEquals(Collections.emptyList(), jobs.active()); assertEquals(runId, jobs.last(id, systemTest).get().id()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 4fc952b0b15..9cb40d60677 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -1,9 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -11,14 +12,17 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Test; import java.time.Duration; +import java.util.List; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; @@ -213,6 +217,51 @@ public class MetricsReporterTest { assertEquals("Queue consumed", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); } + @Test + public void test_nodes_failing_system_upgrade() { + var tester = new DeploymentTester(); + var reporter = createReporter(tester.controller()); + var zone1 = ZoneApiMock.fromId("prod.eu-west-1"); + tester.controllerTester().zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone1)); + var systemUpgrader = new SystemUpgrader(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controllerTester().curator())); + tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer); + + // System on initial version + var version0 = Version.fromString("7.0"); + tester.upgradeSystem(version0); + reporter.maintain(); + assertEquals(0, getNodesFailingUpgrade()); + + for (var version : List.of(Version.fromString("7.1"), Version.fromString("7.2"))) { + // System starts upgrading to next version + tester.upgradeController(version); + reporter.maintain(); + assertEquals(0, getNodesFailingUpgrade()); + systemUpgrader.maintain(); + + // 30 minutes pass and nothing happens + tester.clock().advance(Duration.ofMinutes(30)); + tester.computeVersionStatus(); + reporter.maintain(); + assertEquals(0, getNodesFailingUpgrade()); + + // 1/3 nodes upgrade within timeout + tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version, 1); + tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + tester.computeVersionStatus(); + reporter.maintain(); + assertEquals(2, getNodesFailingUpgrade()); + + // 3/3 nodes upgrade + tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version); + tester.computeVersionStatus(); + reporter.maintain(); + assertEquals(0, getNodesFailingUpgrade()); + assertEquals(version, tester.controller().systemVersion()); + } + } + private Duration getAverageDeploymentDuration(ApplicationId id) { return Duration.ofSeconds(getMetric(MetricsReporter.DEPLOYMENT_AVERAGE_DURATION, id).longValue()); } @@ -225,6 +274,10 @@ public class MetricsReporterTest { return getMetric(MetricsReporter.DEPLOYMENT_WARNINGS, id).intValue(); } + private int getNodesFailingUpgrade() { + return metrics.getMetric(MetricsReporter.NODES_FAILING_SYSTEM_UPGRADE).intValue(); + } + private Number getMetric(String name, ApplicationId id) { return metrics.getMetric((dimensions) -> id.tenant().value().equals(dimensions.get("tenant")) && appDimension(id).equals(dimensions.get("app")), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 9677df6fd18..72b26aca588 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -1,10 +1,9 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -59,6 +58,7 @@ public class SystemUpgraderTest { systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.configServer, version1, zone1, zone2, zone3, zone4); assertCurrentVersion(SystemApplication.proxy, version1, zone1, zone2, zone3, zone4); + assertSystemVersion(version1); // Controller upgrades Version version2 = Version.fromString("6.6"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 3ba1181f762..08963b9fec7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -1,13 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.ImmutableBiMap; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.security.KeyUtils; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; @@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -37,6 +36,7 @@ import org.junit.Test; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.PublicKey; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -48,7 +48,6 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.stream.Collectors; import static com.yahoo.config.provision.SystemName.main; import static java.util.Optional.empty; @@ -64,6 +63,15 @@ public class ApplicationSerializerTest { private static final Path testData = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/"); private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); private static final ZoneId zone2 = ZoneId.from("prod", "us-east-3"); + private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"); + private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + + "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + + "-----END PUBLIC KEY-----\n"); + @Test public void testSerialization() { @@ -134,7 +142,7 @@ public class ApplicationSerializerTest { Optional.of(User.from("by-username")), OptionalInt.of(7), new ApplicationMetrics(0.5, 0.9), - Set.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), + Set.of(publicKey, otherPublicKey), projectId, true, instances); @@ -178,17 +186,14 @@ public class ApplicationSerializerTest { assertEquals(original.owner(), serialized.owner()); assertEquals(original.majorVersion(), serialized.majorVersion()); assertEquals(original.change(), serialized.change()); - assertEquals(original.pemDeployKeys(), serialized.pemDeployKeys()); + assertEquals(original.deployKeys(), serialized.deployKeys()); assertEquals(original.require(id1.instance()).rotations(), serialized.require(id1.instance()).rotations()); assertEquals(original.require(id1.instance()).rotationStatus(), serialized.require(id1.instance()).rotationStatus()); // Test cluster utilization assertEquals(0, serialized.require(id1.instance()).deployments().get(zone1).clusterUtils().size()); - assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().size()); - assertEquals(0.4, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id2")).getCpu(), 0.01); - assertEquals(0.2, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getCpu(), 0.01); - assertEquals(0.2, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getMemory(), 0.01); + assertEquals(0, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().size()); // Test cluster info assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 51df0e4b08b..ff1c952c2a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence;// Copyright 2018 Yahoo Ho import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; +import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; @@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Test; import java.net.URI; +import java.security.PublicKey; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -29,6 +31,14 @@ import static org.junit.Assert.assertTrue; public class TenantSerializerTest { private static final TenantSerializer serializer = new TenantSerializer(); + private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"); + private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + + "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + + "-----END PUBLIC KEY-----\n"); @Test public void athenz_tenant() { @@ -78,12 +88,12 @@ public class TenantSerializerTest { public void cloud_tenant() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), new BillingInfo("old cat lady", "vespa"), - ImmutableBiMap.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", new SimplePrincipal("joe"), - "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----", new SimplePrincipal("jane"))); + ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), + otherPublicKey, new SimplePrincipal("jane"))); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.name(), serialized.name()); assertEquals(tenant.billingInfo(), serialized.billingInfo()); - assertEquals(tenant.pemDeveloperKeys(), serialized.pemDeveloperKeys()); + assertEquals(tenant.developerKeys(), serialized.developerKeys()); } private Contact contact() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index a1e22b4fc64..5d65cf0381e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -1,20 +1,23 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; @@ -36,9 +39,11 @@ public class VersionStatusSerializerTest { ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false, - true, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + true, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), + Instant.ofEpochMilli(123), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, true, - false, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + false, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), + Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); VersionStatus status = new VersionStatus(vespaVersions); VersionStatusSerializer serializer = new VersionStatusSerializer(); VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); @@ -53,14 +58,48 @@ public class VersionStatusSerializerTest { assertEquals(a.isSystemVersion(), b.isSystemVersion()); assertEquals(a.isReleased(), b.isReleased()); assertEquals(a.statistics(), b.statistics()); - assertEquals(a.systemApplicationHostnames(), b.systemApplicationHostnames()); + assertEquals(a.nodeVersions(), b.nodeVersions()); assertEquals(a.confidence(), b.confidence()); } } - private static List<HostName> asHostnames(String... hostname) { - return Arrays.stream(hostname).map(HostName::from).collect(Collectors.toList()); + @Test + public void testLegacySerialization() throws Exception { + var data = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json")); + var serializer = new VersionStatusSerializer(); + var deserializedStatus = serializer.fromSlime(SlimeUtils.jsonToSlime(data)); + + var statistics = new DeploymentStatistics( + Version.fromString("7.0"), + List.of(), + List.of(), + List.of() + ); + var vespaVersion = new VespaVersion(statistics, "badc0ffee", + Instant.ofEpochMilli(123), true, + true, true, + nodeVersions(Version.emptyVersion, Version.emptyVersion, + Instant.EPOCH, "cfg1", "cfg2", "cfg3"), + VespaVersion.Confidence.normal); + + VespaVersion deserialized = deserializedStatus.versions().get(0); + assertEquals(vespaVersion.releaseCommit(), deserialized.releaseCommit()); + assertEquals(vespaVersion.committedAt().truncatedTo(MILLIS), deserialized.committedAt()); + assertEquals(vespaVersion.isControllerVersion(), deserialized.isControllerVersion()); + assertEquals(vespaVersion.isSystemVersion(), deserialized.isSystemVersion()); + assertEquals(vespaVersion.isReleased(), deserialized.isReleased()); + assertEquals(vespaVersion.statistics(), deserialized.statistics()); + assertEquals(vespaVersion.nodeVersions(), deserialized.nodeVersions()); + assertEquals(vespaVersion.confidence(), deserialized.confidence()); + } + + private static NodeVersions nodeVersions(Version version, Version wantedVersion, Instant changedAt, String... hostnames) { + var nodeVersions = new ArrayList<NodeVersion>(); + for (var hostname : hostnames) { + nodeVersions.add(new NodeVersion(HostName.from(hostname), version, wantedVersion, changedAt)); + } + return NodeVersions.EMPTY.with(nodeVersions); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json index 8ab277a3795..1c660726d61 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json @@ -17,11 +17,11 @@ "queryQuality": 100, "writeQuality": 99.99894341115082, "pemDeployKeys": [ - "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----" + "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----" ], "pemDeveloperKeys": [ { - "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----", + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----", "user": "joe@dev" } ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json new file mode 100644 index 00000000000..96ca22e1c1a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json @@ -0,0 +1,23 @@ +{ + "versions": [ + { + "releaseCommit": "badc0ffee", + "releasedAt": 123, + "isCurrentControllerVersion": true, + "isCurrentSystemVersion": true, + "isReleased": true, + "deploymentStatistics": { + "version": "7.0", + "failing": [], + "production": [], + "deploying": [] + }, + "confidence": "normal", + "configServerHostnames": [ + "cfg1", + "cfg2", + "cfg3" + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 2d8c937097a..80e52f373d7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -79,8 +79,10 @@ public class ContainerControllerTester { Optional.of(new PropertyId("1234"))); controller().tenants().create(tenantSpec, credentials); - ApplicationId app = ApplicationId.from(tenant, application, instance); - return controller().applications().createApplication(app, Optional.of(credentials)); + TenantAndApplicationId id = TenantAndApplicationId.from(tenant, application); + controller().applications().createApplication(id, Optional.of(credentials)); + controller().applications().createInstance(id.instance(instance)); + return controller().applications().requireApplication(id); } public void deploy(ApplicationId id, ApplicationPackage applicationPackage, ZoneId zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 3ff21bb2261..307496ace5a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -29,6 +29,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -49,11 +52,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; @@ -111,6 +111,11 @@ import static org.junit.Assert.assertTrue; public class ApplicationApiTest extends ControllerContainerTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/"; + private static final String pemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"; + private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n"); private static final ApplicationPackage applicationPackageDefault = new ApplicationPackageBuilder() .instances("default") @@ -320,7 +325,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .region("us-west-1") .build(); - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", POST) + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference-2.json")); @@ -360,14 +365,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a pem deploy key tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", POST) .userIdentity(USER_ID) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + "{\"keys\":[\"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\\n-----END PUBLIC KEY-----\\n\"]}"); // PATCH in a pem deploy key at deprecated path tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", PATCH) .userIdentity(USER_ID) - .data("{\"pemDeployKey\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"); + .data("{\"pemDeployKey\":\"" + pemPublicKey + "\"}"), + "{\"message\":\"Added deploy key " + quotedPemPublicKey + "\"}"); // GET an application with a major version override tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET) @@ -383,8 +388,8 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE the pem deploy key tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", DELETE) .userIdentity(USER_ID) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Removed deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + "{\"keys\":[]}"); tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET) .userIdentity(USER_ID), @@ -556,6 +561,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .oktaAccessToken(OKTA_AT), new File("delete-with-active-deployments.json"), 400); + // GET test-config for local tests against a prod deployment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/test-config", GET) + .userIdentity(USER_ID), + new File("test-config.json")); + // DELETE (deactivate) a deployment - dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE) .userIdentity(USER_ID), @@ -1061,7 +1071,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .oktaAccessToken(OKTA_AT) .userIdentity(USER_ID), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1.instance1': Application already exists\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1.instance1': Instance already exists\"}", 400); ConfigServerMock configServer = serviceRegistry().configServerMock(); @@ -1111,7 +1121,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .oktaAccessToken(OKTA_AT) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1.instance1': Application not found\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete instance 'tenant1.application1.instance1': Instance not found\"}", 404); // DELETE tenant diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json index 331aabd32d0..9d76654fbc0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json @@ -80,8 +80,8 @@ "majorVersion": 7, "globalRotations": [], "instances": [], - "pemDeployKey": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----", - "pemDeployKeys": ["-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"], + "pemDeployKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", + "pemDeployKeys": ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n"], "metrics": { "queryServiceQuality": 0.0, "writeServiceQuality": 0.0 diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 25948e998f1..d62e39e42e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -28,40 +28,10 @@ "lastWritesPerSecond": 2.0 }, "cost": { - "tco": 74, + "tco": 0, "waste": 0, - "utilization": 2.9999999999999996, - "cluster": { - "cluster1": { - "count": 2, - "resource": "cpu", - "utilization": 2.9999999999999996, - "tco": 74, - "waste": 0, - "flavor": "flavor1", - "flavorCost":37.0, - "flavorCpu":2.0, - "flavorMem":4.0, - "flavorDisk":50.0, - "type": "content", - "util": { - "cpu": 2.9999999999999996, - "mem": 0.4285714285714286, - "disk": 0.5714285714285715, - "diskBusy": 1.0 - }, - "usage": { - "cpu": 0.6, - "mem": 0.3, - "disk": 0.4, - "diskBusy": 0.3 - }, - "hostnames": [ - "host1", - "host2" - ] - } - } + "utilization": 0.0, + "cluster": {} }, "metrics": { "queriesPerSecond": 1.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 1a2025e4de2..c56a269b9d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -25,40 +25,10 @@ "lastWritesPerSecond": 2.0 }, "cost": { - "tco": 74, + "tco": 0, "waste": 0, - "utilization": 2.9999999999999996, - "cluster": { - "cluster1": { - "count": 2, - "resource": "cpu", - "utilization": 2.9999999999999996, - "tco": 74, - "waste": 0, - "flavor": "flavor1", - "flavorCost": 37.0, - "flavorCpu": 2.0, - "flavorMem": 4.0, - "flavorDisk": 50.0, - "type": "content", - "util": { - "cpu": 2.9999999999999996, - "mem": 0.4285714285714286, - "disk": 0.5714285714285715, - "diskBusy": 1.0 - }, - "usage": { - "cpu": 0.6, - "mem": 0.3, - "disk": 0.4, - "diskBusy": 0.3 - }, - "hostnames": [ - "host1", - "host2" - ] - } - } + "utilization": 0.0, + "cluster": {} }, "metrics": { "queriesPerSecond": 1.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index bb68904bee6..140be562fe9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -37,40 +37,10 @@ "lastWritesPerSecond": 2.0 }, "cost": { - "tco": 74, + "tco": 0, "waste": 0, - "utilization": 2.9999999999999996, - "cluster": { - "cluster1": { - "count": 2, - "resource": "cpu", - "utilization": 2.9999999999999996, - "tco": 74, - "waste": 0, - "flavor": "flavor1", - "flavorCost": 37.0, - "flavorCpu": 2.0, - "flavorMem": 4.0, - "flavorDisk": 50.0, - "type": "content", - "util": { - "cpu": 2.9999999999999996, - "mem": 0.4285714285714286, - "disk": 0.5714285714285715, - "diskBusy": 1.0 - }, - "usage": { - "cpu": 0.6, - "mem": 0.3, - "disk": 0.4, - "diskBusy": 0.3 - }, - "hostnames": [ - "host1", - "host2" - ] - } - } + "utilization": 0.0, + "cluster": {} }, "metrics": { "queriesPerSecond": 1.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json new file mode 100644 index 00000000000..2338543b019 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json @@ -0,0 +1,20 @@ +{ + "application": "tenant1:application1:instance1", + "zone": "prod.us-central-1", + "system": "main", + "endpoints": { + "prod.us-central-1": [ + "http://old-endpoint.vespa.yahooapis.com:4080" + ] + }, + "zoneEndpoints": { + "prod.us-central-1": { + "default": "http://old-endpoint.vespa.yahooapis.com:4080" + } + }, + "clusters": { + "prod.us-central-1": [ + "music" + ] + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 084b235943e..0a4d046e318 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -1,26 +1,26 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; import java.io.File; +import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; /** * @author bratseth @@ -69,16 +69,15 @@ public class DeploymentApiTest extends ControllerContainerTest { private VersionStatus censorConfigServers(VersionStatus versionStatus, Controller controller) { List<VespaVersion> censored = new ArrayList<>(); for (VespaVersion version : versionStatus.versions()) { - if (!version.systemApplicationHostnames().isEmpty()) { + if (version.nodeVersions().size() > 0) { version = new VespaVersion(version.statistics(), version.releaseCommit(), version.committedAt(), version.isControllerVersion(), version.isSystemVersion(), version.isReleased(), - ImmutableSet.of("config1.test", "config2.test").stream() - .map(HostName::from) - .collect(Collectors.toSet()), + NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("config1.test"), version.versionNumber(), version.versionNumber(), Instant.EPOCH), + new NodeVersion(HostName.from("config2.test"), version.versionNumber(), version.versionNumber(), Instant.EPOCH))), VespaVersion.confidenceFrom(version.statistics(), controller) ); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 3b7d55f8cef..0a1e996696b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -3,15 +3,21 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import ai.vespa.hosted.api.Method; import ai.vespa.hosted.api.RequestSigner; +import com.google.common.collect.ImmutableBiMap; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.security.KeyUtils; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import org.junit.Before; import org.junit.Test; @@ -19,6 +25,8 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.net.URI; import java.net.http.HttpRequest; +import java.security.PrivateKey; +import java.security.PublicKey; import java.util.Set; import static org.junit.Assert.assertEquals; @@ -27,21 +35,21 @@ import static org.junit.Assert.assertTrue; public class SignatureFilterTest { - private static final String publicKey = "-----BEGIN PUBLIC KEY-----\n" + - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + - "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + - "-----END PUBLIC KEY-----\n"; + private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"); - private static final String otherPublicKey = "-----BEGIN PUBLIC KEY-----\n" + - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + - "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + - "-----END PUBLIC KEY-----\n"; + private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + + "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + + "-----END PUBLIC KEY-----\n"); - private static final String privateKey = "-----BEGIN EC PRIVATE KEY-----\n" + - "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" + - "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" + - "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + - "-----END EC PRIVATE KEY-----\n"; + private static final PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey("-----BEGIN EC PRIVATE KEY-----\n" + + "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" + + "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" + + "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END EC PRIVATE KEY-----\n"); private static final TenantAndApplicationId appId = TenantAndApplicationId.from("my-tenant", "my-app"); private static final ApplicationId id = appId.defaultInstance(); @@ -58,10 +66,10 @@ public class SignatureFilterTest { filter = new SignatureFilter(tester.controller()); signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock()); - tester.createApplication(tester.createTenant(id.tenant().value(), "unused", 496L), - id.application().value(), - id.instance().value(), - 28L); + tester.curator().writeTenant(new CloudTenant(appId.tenant(), + new BillingInfo("id", "code"), + ImmutableBiMap.of())); + tester.curator().writeApplication(new Application(appId, tester.clock().instant())); } @Test @@ -69,42 +77,57 @@ public class SignatureFilterTest { // Unsigned request gets no role. HttpRequest.Builder request = HttpRequest.newBuilder(URI.create("https://host:123/path/./..//..%2F?query=empty&%3F=%26")); byte[] emptyBody = new byte[0]; - DiscFilterRequest unsigned = requestOf(request.method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody); - filter.filter(unsigned); - assertNull(unsigned.getAttribute(SecurityContext.ATTRIBUTE_NAME)); + verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody), + null); // Signed request gets no role when no key is stored for the application. - DiscFilterRequest signed = requestOf(signer.signed(request, Method.GET, InputStream::nullInputStream), emptyBody); - filter.filter(signed); - assertNull(signed.getAttribute(SecurityContext.ATTRIBUTE_NAME)); - - // Signed request gets no role when a non-matching key is stored for the application. - applications.lockApplicationOrThrow(appId, application -> applications.store(application.withPemDeployKey(otherPublicKey))); - filter.filter(signed); - assertNull(signed.getAttribute(SecurityContext.ATTRIBUTE_NAME)); - - // Signed request gets a build service role when a matching key is stored for the application. - applications.lockApplicationOrThrow(appId, application -> applications.store(application.withPemDeployKey(publicKey))); - assertTrue(filter.filter(signed).isEmpty()); - SecurityContext securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME); - assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName()); - assertEquals(Set.of(Role.buildService(id.tenant(), id.application()), - Role.applicationDeveloper(id.tenant(), id.application())), - securityContext.roles()); - - // Signed POST request also gets a build service role. + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), + null); + + // Signed request gets no role when only non-matching keys are stored for the application. + applications.lockApplicationOrThrow(appId, application -> applications.store(application.withDeployKey(otherPublicKey))); + // Signed request gets no role when no key is stored for the application. + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), + null); + + // Signed request gets a headless role when a matching key is stored for the application. + applications.lockApplicationOrThrow(appId, application -> applications.store(application.withDeployKey(publicKey))); + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), + new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); // TODO jonmv: Change to headless. + + // TODO jonmv: remove after Oct 2019. + // Signed request gets a build service role when a matching key is stored for the application and no X-Key header is provided. + verifySecurityContext(requestOf(signer.legacySigned(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), + new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); + + // Signed POST request with X-Key header gets a headless role. byte[] hiBytes = new byte[]{0x48, 0x69}; - signed = requestOf(signer.signed(request, Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes); - filter.filter(signed); - securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME); - assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName()); - assertEquals(Set.of(Role.buildService(id.tenant(), id.application()), - Role.applicationDeveloper(id.tenant(), id.application())), - securityContext.roles()); + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), + new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); // TODO jonmv: Change to headless. + + // Signed request gets a developer role when a matching developer key is stored for the tenant. + tester.curator().writeTenant(new CloudTenant(appId.tenant(), + new BillingInfo("id", "code"), + ImmutableBiMap.of(publicKey, () -> "user"))); + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), + new SecurityContext(new SimplePrincipal("user"), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); // Unsigned requests still get no roles. - filter.filter(unsigned); - assertNull(unsigned.getAttribute(SecurityContext.ATTRIBUTE_NAME)); + verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody), + null); + } + + private void verifySecurityContext(DiscFilterRequest request, SecurityContext securityContext) { + assertTrue(filter.filter(request).isEmpty()); + assertEquals(securityContext, request.getAttribute(SecurityContext.ATTRIBUTE_NAME)); } private static DiscFilterRequest requestOf(HttpRequest request, byte[] body) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json index 17f90259fa8..01af1bd70dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json @@ -6,92 +6,92 @@ "cloud": "cloud1", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-2-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-1-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-1-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-3-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" } @@ -103,47 +103,47 @@ "cloud": "cloud2", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-2-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-1-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-3-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-2-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json index 86bc272fcd1..dbaa6623fae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json @@ -6,92 +6,92 @@ "cloud": "cloud1", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-2-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-1-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-1-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-3-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" } @@ -103,47 +103,47 @@ "cloud": "cloud2", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-2-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-1-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-3-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-2-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json index e8007fbf6c5..2b907c1156c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json @@ -6,47 +6,47 @@ "cloud": "cloud1", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-1-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-3-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" } @@ -58,47 +58,47 @@ "cloud": "cloud1", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-2-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-1-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" } @@ -110,47 +110,47 @@ "cloud": "cloud2", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-2-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-1-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-3-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-2-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index b17dda7f810..b1f5f33b960 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -12,7 +12,6 @@ import java.io.File; import java.util.Set; import static com.yahoo.application.container.handler.Request.Method.DELETE; -import static com.yahoo.application.container.handler.Request.Method.PATCH; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; import static org.junit.Assert.assertEquals; @@ -23,6 +22,17 @@ import static org.junit.Assert.assertEquals; public class UserApiTest extends ControllerContainerCloudTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/"; + private static final String pemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"; + private static final String otherPemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + + "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + + "-----END PUBLIC KEY-----\n"; + private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n"); + private static final String otherQuotedPemPublicKey = otherPemPublicKey.replaceAll("\\n", "\\\\n"); + @Test public void testUserManagement() { @@ -132,30 +142,30 @@ public class UserApiTest extends ControllerContainerCloudTest { // POST a pem deploy key tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app/key", POST) .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + new File("first-deploy-key.json")); // POST a pem developer key tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) .user("joe@dev") .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY----- for joe@dev\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + new File("first-developer-key.json")); // POST the same pem developer key for a different user is forbidden tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) .user("operator@tenant") .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----\"}"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Multiple entries with same key: -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----=operator@tenant and -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----=joe@dev\"}", + .data("{\"key\":\"" + pemPublicKey + "\"}"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Key "+ quotedPemPublicKey + " is already owned by joe@dev\"}", 400); // PATCH in a different pem developer key tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) .user("operator@tenant") .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\nƪ(`▿▿▿▿´ƪ)\\n-----END PUBLIC KEY----- for operator@tenant\"}"); + .data("{\"key\":\"" + otherPemPublicKey + "\"}"), + new File("both-developer-keys.json")); // GET tenant information with keys tester.assertResponse(request("/application/v4/tenant/my-tenant/") @@ -165,8 +175,8 @@ public class UserApiTest extends ControllerContainerCloudTest { // DELETE a pem developer key tester.assertResponse(request("/application/v4/tenant/my-tenant/key", DELETE) .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Removed developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)＿\\n-----END PUBLIC KEY----- for joe@dev\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + new File("second-developer-key.json")); // DELETE an application role is allowed for an application admin. tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", DELETE) @@ -180,8 +190,7 @@ public class UserApiTest extends ControllerContainerCloudTest { "{\"message\":\"Deleted application my-tenant.my-app\"}"); // DELETE a tenant role is available to tenant admins. - // DELETE the tenantOperator role clears any developer key. - // TODO jonmv: Change to developer, when this role exists. + // DELETE the developer role clears any developer key. tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE) .roles(Set.of(Role.tenantAdmin(id.tenant()))) .data("{\"user\":\"operator@tenant\",\"roleName\":\"tenantOperator\"}"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json index 6cf4dc76173..31bdb07b26b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json @@ -1,6 +1,5 @@ { "tenant": "my-tenant", "application": "my-app", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app/instance/default" + "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json new file mode 100644 index 00000000000..2ff1c29fe29 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json @@ -0,0 +1,12 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", + "user": "joe@dev" + }, + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", + "user": "operator@tenant" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json new file mode 100644 index 00000000000..1c86877b77d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json @@ -0,0 +1,5 @@ +{ + "keys": [ + "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n" + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json new file mode 100644 index 00000000000..b7d48f283f3 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json @@ -0,0 +1,9 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", + "user": "joe@dev" + } + ] +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json new file mode 100644 index 00000000000..f7d90f31116 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json @@ -0,0 +1,8 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", + "user": "operator@tenant" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 5aaa900c3f0..b7970a48963 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -1,27 +1,14 @@ { "tenant": "my-tenant", "type": "CLOUD", - "pemDeployKeys": [ - { - "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----", - "application": "my-app" - } - ], "pemDeveloperKeys": [ { - "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----", + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", "user": "joe@dev" }, { - "key": "-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----", + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", "user": "operator@tenant" }], - "applications": [ - { - "tenant":"my-tenant", - "application":"my-app", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/my-tenant/application/my-app/instance/default" - } - ] + "applications": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json index a89a0f5360c..39b6cccbab0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -1,7 +1,6 @@ { "tenant": "my-tenant", "type": "CLOUD", - "pemDeployKeys": [], "pemDeveloperKeys": [], "applications": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index ba8309de286..83223b0e041 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java index 61893a30e7e..0ca1b3e5603 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java @@ -38,8 +38,8 @@ public class Properties { return Paths.get(requireNonBlankProperty("privateKeyFile")); } - public static Path certificateFile() { - return Paths.get(requireNonBlankProperty("certificateFile")); + public static Optional<Path> certificateFile() { + return getNonBlankProperty("certificateFile").map(Paths::get); } /** Returns the system property with the given name if it is set, or empty. */ diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java index b2fd16b7975..5d314d90356 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java @@ -4,9 +4,10 @@ package ai.vespa.hosted.api; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureUtils; -import java.io.ByteArrayInputStream; import java.io.InputStream; import java.net.http.HttpRequest; +import java.security.PrivateKey; +import java.security.PublicKey; import java.security.Signature; import java.security.SignatureException; import java.time.Clock; @@ -15,6 +16,7 @@ import java.util.function.Supplier; import static ai.vespa.hosted.api.Signatures.sha256Digest; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static java.nio.charset.StandardCharsets.UTF_8; /** * Signs HTTP request headers using a private key, for verification by the indicated public key. @@ -25,6 +27,7 @@ public class RequestSigner { private final Signature signer; private final String keyId; + private final String base64PemPublicKey; private final Clock clock; /** Creates a new request signer from the given PEM encoded ECDSA key, with a public key with the given ID. */ @@ -34,8 +37,15 @@ public class RequestSigner { /** Creates a new request signer with a custom clock. */ public RequestSigner(String pemPrivateKey, String keyId, Clock clock) { - this.signer = SignatureUtils.createSigner(KeyUtils.fromPemEncodedPrivateKey(pemPrivateKey), SHA256_WITH_ECDSA); + this(KeyUtils.fromPemEncodedPrivateKey(pemPrivateKey), keyId, clock); + } + + /** Creates a new request signer with a custom clock. */ + public RequestSigner(PrivateKey privateKey, String keyId, Clock clock) { + this.signer = SignatureUtils.createSigner(privateKey, SHA256_WITH_ECDSA); this.keyId = keyId; + this.base64PemPublicKey = Base64.getEncoder().encodeToString(KeyUtils.toPem(KeyUtils.extractPublicKey(privateKey)).getBytes(UTF_8)); + PublicKey key = KeyUtils.extractPublicKey(privateKey); this.clock = clock; } @@ -44,8 +54,8 @@ public class RequestSigner { * <br> * The request builder's method and data are set to the given arguments, and a hash of the * content is computed and added to a header, together with other meta data, like the URI - * of the request, the current UTC time, and the id of the public key which shall be used to - * verify this signature. + * of the request, the current UTC time, and the id and value of the public key which shall + * be used to * verify this signature. * Finally, a signature is computed from these fields, based on the private key of this, and * added to the request as another header. */ @@ -60,6 +70,29 @@ public class RequestSigner { request.setHeader("X-Timestamp", timestamp); request.setHeader("X-Content-Hash", contentHash); request.setHeader("X-Key-Id", keyId); + request.setHeader("X-Key", base64PemPublicKey); + request.setHeader("X-Authorization", signature); + + request.method(method.name(), HttpRequest.BodyPublishers.ofInputStream(data)); + return request.build(); + } + catch (SignatureException e) { + throw new IllegalArgumentException(e); + } + } + + // TODO jonmv: Simulates old clients — remove shortly (2 Oct 2019). + public HttpRequest legacySigned(HttpRequest.Builder request, Method method, Supplier<InputStream> data) { + try { + String timestamp = clock.instant().toString(); + String contentHash = Base64.getEncoder().encodeToString(sha256Digest(data::get)); + byte[] canonicalMessage = Signatures.canonicalMessageOf(method.name(), request.copy().build().uri(), timestamp, contentHash); + signer.update(canonicalMessage); + String signature = Base64.getEncoder().encodeToString(signer.sign()); + + request.setHeader("X-Timestamp", timestamp); + request.setHeader("X-Content-Hash", contentHash); + request.setHeader("X-Key-Id", keyId); request.setHeader("X-Authorization", signature); request.method(method.name(), HttpRequest.BodyPublishers.ofInputStream(data)); diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java index 9d85ec9bf6b..5a6bea54bce 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java @@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureUtils; import java.net.URI; +import java.security.PublicKey; import java.security.Signature; import java.security.SignatureException; import java.time.Clock; @@ -31,7 +32,12 @@ public class RequestVerifier { /** Creates a new request verifier from the given PEM encoded ECDSA public key, with the given clock. */ public RequestVerifier(String pemPublicKey, Clock clock) { - this.verifier = SignatureUtils.createVerifier(KeyUtils.fromPemEncodedPublicKey(pemPublicKey), SHA256_WITH_ECDSA); + this(KeyUtils.fromPemEncodedPublicKey(pemPublicKey), clock); + } + + /** Creates a new request verifier from the given PEM encoded ECDSA public key, with the given clock. */ + public RequestVerifier(PublicKey publicKey, Clock clock) { + this.verifier = SignatureUtils.createVerifier(publicKey, SHA256_WITH_ECDSA); this.clock = clock; } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java b/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java index b8698eab15f..c1104c649f2 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/TestConfig.java @@ -3,13 +3,16 @@ package ai.vespa.hosted.api; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.JsonDecoder; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import java.net.URI; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -26,8 +29,10 @@ public class TestConfig { private final ZoneId zone; private final SystemName system; private final Map<ZoneId, Map<String, URI>> deployments; + private final Map<ZoneId, List<String>> contentClusters; - public TestConfig(ApplicationId application, ZoneId zone, SystemName system, Map<ZoneId, Map<String, URI>> deployments) { + public TestConfig(ApplicationId application, ZoneId zone, SystemName system, Map<ZoneId, Map<String, URI>> deployments, + Map<ZoneId, List<String>> contentClusters) { if ( ! deployments.containsKey(zone)) throw new IllegalArgumentException("Config must contain a deployment for its zone, but only does for " + deployments.keySet()); this.application = requireNonNull(application); @@ -36,12 +41,15 @@ public class TestConfig { this.deployments = deployments.entrySet().stream() .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey(), entry -> Map.copyOf(entry.getValue()))); + this.contentClusters = contentClusters.entrySet().stream() + .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey(), + entry -> List.copyOf(entry.getValue()))); } /** * Parses the given test config JSON and returns a new config instance. * - * If the given JSON has a "clusters" element, a config object with default values + * If the given JSON has a "localEndpoints" element, a config object with default values * is returned, using {@link #fromEndpointsOnly}. Otherwise, all config attributes are parsed. */ public static TestConfig fromJson(byte[] jsonBytes) { @@ -56,7 +64,13 @@ public class TestConfig { config.field("zoneEndpoints").traverse((ObjectTraverser) (zoneId, clustersObject) -> { deployments.put(ZoneId.from(zoneId), toClusterMap(clustersObject)); }); - return new TestConfig(application, zone, system, deployments); + Map<ZoneId, List<String>> contentClusters = new HashMap<>(); + config.field("clusters").traverse(((ObjectTraverser) (zoneId, clustersArray) -> { + List<String> clusters = new ArrayList<>(); + clustersArray.traverse((ArrayTraverser) (__, cluster) -> clusters.add(cluster.asString())); + contentClusters.put(ZoneId.from(zoneId), clusters); + })); + return new TestConfig(application, zone, system, deployments, contentClusters); } static Map<String, URI> toClusterMap(Inspector clustersObject) { @@ -73,7 +87,8 @@ public class TestConfig { return new TestConfig(ApplicationId.defaultId(), ZoneId.defaultId(), SystemName.defaultSystem(), - Map.of(ZoneId.defaultId(), endpoints)); + Map.of(ZoneId.defaultId(), endpoints), + Map.of()); } /** Returns the full id of the application to test. */ @@ -85,6 +100,9 @@ public class TestConfig { /** Returns an immutable view of deployments, per zone, of the application to test. */ public Map<ZoneId, Map<String, URI>> deployments() { return deployments; } + /** Returns an immutable view of content clusters, per zone, of the application to test. */ + public Map<ZoneId, List<String>> contentClusters() { return contentClusters; } + /** Returns the hosted Vespa system this is run against. */ public SystemName system() { return system; } diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java index bad838f0579..5ed008cc2ec 100644 --- a/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java +++ b/hosted-api/src/test/java/ai/vespa/hosted/api/TestConfigTest.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -32,6 +33,9 @@ public class TestConfigTest { ZoneId.from("prod", "aws-us-east-1a"), Map.of("default", URI.create("https://prod.endpoint:443/"))), config.deployments()); + assertEquals(Map.of(ZoneId.from("prod", "aws-us-east-1c"), + List.of("documents")), + config.contentClusters()); } @Test diff --git a/hosted-api/src/test/resources/test-config.json b/hosted-api/src/test/resources/test-config.json index 9d36f9496a0..bd337e1c28a 100644 --- a/hosted-api/src/test/resources/test-config.json +++ b/hosted-api/src/test/resources/test-config.json @@ -9,5 +9,10 @@ "prod.aws-us-east-1a": { "default": "https://prod.endpoint:443/" } + }, + "clusters": { + "prod.aws-us-east-1c": [ + "documents" + ] } } diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index ca035ec1e2e..0bedd1d1704 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -104,11 +104,6 @@ </exclusions> </dependency> <dependency> - <groupId>commons-daemon</groupId> - <artifactId>commons-daemon</artifactId> - <scope>compile</scope> - </dependency> - <dependency> <groupId>org.apache.felix</groupId> <artifactId>org.apache.felix.framework</artifactId> <scope>compile</scope> @@ -243,7 +238,6 @@ <classpath /> <argument>com.yahoo.jdisc.core.ExportPackages</argument> <argument>${exportPackagesFile}</argument> - <argument>${project.build.directory}/dependency/commons-daemon.jar</argument> <argument>__REPLACE_VERSION__${project.build.directory}/dependency/guava.jar</argument> <argument>${project.build.directory}/dependency/guice-no_aop.jar</argument> <argument>${project.build.directory}/dependency/guice-assistedinject.jar</argument> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapDaemon.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapDaemon.java deleted file mode 100644 index de6d5c5073f..00000000000 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapDaemon.java +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.core; - -import com.yahoo.protect.Process; -import org.apache.commons.daemon.Daemon; -import org.apache.commons.daemon.DaemonContext; - -import java.util.Arrays; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author Simon Thoresen Hult - */ -public class BootstrapDaemon implements Daemon { - - private static final Logger log = Logger.getLogger(BootstrapDaemon.class.getName()); - private final BootstrapLoader loader; - private final boolean privileged; - private String bundleLocation; - - static { - // force load slf4j to avoid other logging frameworks from initializing before it - org.slf4j.LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); - } - - public BootstrapDaemon() { - this(new ApplicationLoader(Main.newOsgiFramework(), Main.newConfigModule()), - Boolean.valueOf(System.getProperty("jdisc.privileged"))); - } - - BootstrapDaemon(BootstrapLoader loader, boolean privileged) { - this.loader = loader; - this.privileged = privileged; - } - - BootstrapLoader loader() { - return loader; - } - - private static class WatchDog implements Runnable { - final String name; - final CountDownLatch complete; - final long timeout; - final TimeUnit timeUnit; - WatchDog(String name, CountDownLatch complete, long timeout, TimeUnit timeUnit) { - this.name = name; - this.complete = complete; - this.timeout = timeout; - this.timeUnit = timeUnit; - } - @Override - public void run() { - boolean dumpStack; - try { - dumpStack = !complete.await(timeout, timeUnit); - } catch (InterruptedException e) { - return; - } - if (dumpStack) { - log.warning("The watchdog for BootstrapDaemon." + name + " detected that it had not completed in " - + timeUnit.toMillis(timeout) + "ms. Dumping stack."); - Process.dumpThreads(); - } - } - } - private interface MyRunnable { - void run() throws Exception; - } - private void startWithWatchDog(String name, long timeout, TimeUnit timeUnit, MyRunnable task) throws Exception { - CountDownLatch complete = new CountDownLatch(1); - Thread thread = new Thread(new WatchDog(name, complete, timeout, timeUnit), name); - thread.setDaemon(true); - thread.start(); - try { - task.run(); - } catch (Exception e) { - log.log(Level.WARNING, "Exception caught during BootstrapDaemon." + name, e); - throw e; - } catch (Error e) { - log.log(Level.WARNING, "Error caught during BootstrapDaemon." + name, e); - throw e; - } catch (Throwable thrown) { - log.log(Level.WARNING, "Throwable caught during BootstrapDaemon." + name, thrown); - } finally { - complete.countDown(); - thread.join(); - } - } - - @Override - public void init(DaemonContext context) throws Exception { - String[] args = context.getArguments(); - if (args == null || args.length != 1 || args[0] == null) { - throw new IllegalArgumentException("Expected 1 argument, got " + Arrays.toString(args) + "."); - } - bundleLocation = args[0]; - if (privileged) { - log.finer("Initializing application with privileges."); - startWithWatchDog("init", 60, TimeUnit.SECONDS, () -> loader.init(bundleLocation, true)); - } - } - - @Override - public void start() throws Exception { - try { - if (!privileged) { - log.finer("Initializing application without privileges."); - startWithWatchDog("init", 60, TimeUnit.SECONDS, () -> loader.init(bundleLocation, false)); - } - startWithWatchDog("start", 60, TimeUnit.SECONDS, () -> loader.start()); - } catch (Exception e) { - try { - log.log(Level.SEVERE, "Failed starting container", e); - } - finally { - Runtime.getRuntime().halt(1); - } - } - } - - @Override - public void stop() throws Exception { - startWithWatchDog("stop", 60, TimeUnit.SECONDS, () -> loader.stop()); - } - - @Override - public void destroy() { - try { - startWithWatchDog("destroy", 60, TimeUnit.SECONDS, () -> loader.destroy()); - } catch (Exception e) { - } - } - -} diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/BootstrapDaemonTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/BootstrapDaemonTestCase.java deleted file mode 100644 index df8223a6d86..00000000000 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/BootstrapDaemonTestCase.java +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.core; - -import org.apache.commons.daemon.DaemonContext; -import org.apache.commons.daemon.DaemonController; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author Simon Thoresen Hult - */ -public class BootstrapDaemonTestCase { - - @Test - public void requireThatPrivilegedLifecycleWorks() throws Exception { - MyLoader loader = new MyLoader(); - BootstrapDaemon daemon = new BootstrapDaemon(loader, true); - daemon.init(new MyContext("foo")); - assertTrue(loader.hasState(true, false, false, false)); - assertTrue(loader.privileged); - daemon.start(); - assertTrue(loader.hasState(true, true, false, false)); - daemon.stop(); - assertTrue(loader.hasState(true, true, true, false)); - daemon.destroy(); - assertTrue(loader.hasState(true, true, true, true)); - } - - @Test - public void requireThatNonPrivilegedLifecycleWorks() throws Exception { - MyLoader loader = new MyLoader(); - BootstrapDaemon daemon = new BootstrapDaemon(loader, false); - daemon.init(new MyContext("foo")); - assertTrue(loader.hasState(false, false, false, false)); - daemon.start(); - assertTrue(loader.hasState(true, true, false, false)); - assertFalse(loader.privileged); - daemon.stop(); - assertTrue(loader.hasState(true, true, true, false)); - daemon.destroy(); - assertTrue(loader.hasState(true, true, true, true)); - } - - @Test - public void requireThatBundleLocationIsRequired() throws Exception { - MyLoader loader = new MyLoader(); - BootstrapDaemon daemon = new BootstrapDaemon(loader, true); - try { - daemon.init(new MyContext((String[])null)); - fail(); - } catch (IllegalArgumentException e) { - assertNull(loader.bundleLocation); - } - try { - daemon.init(new MyContext()); - fail(); - } catch (IllegalArgumentException e) { - assertNull(loader.bundleLocation); - } - try { - daemon.init(new MyContext((String)null)); - fail(); - } catch (IllegalArgumentException e) { - assertNull(loader.bundleLocation); - } - try { - daemon.init(new MyContext("foo", "bar")); - fail(); - } catch (IllegalArgumentException e) { - assertNull(loader.bundleLocation); - } - - daemon.init(new MyContext("foo")); - daemon.start(); - - assertNotNull(loader.bundleLocation); - assertEquals("foo", loader.bundleLocation); - - daemon.stop(); - daemon.destroy(); - } - - @Test - public void requireThatEnvironmentIsRequired() { - try { - new BootstrapDaemon(); - fail(); - } catch (IllegalStateException e) { - - } - } - - private static class MyLoader implements BootstrapLoader { - - String bundleLocation = null; - boolean privileged = false; - boolean initCalled = false; - boolean startCalled = false; - boolean stopCalled = false; - boolean destroyCalled = false; - - boolean hasState(boolean initCalled, boolean startCalled, boolean stopCalled, boolean destroyCalled) { - return this.initCalled == initCalled && this.startCalled == startCalled && - this.stopCalled == stopCalled && this.destroyCalled == destroyCalled; - } - - @Override - public void init(String bundleLocation, boolean privileged) throws Exception { - this.bundleLocation = bundleLocation; - this.privileged = privileged; - initCalled = true; - } - - @Override - public void start() throws Exception { - startCalled = true; - } - - @Override - public void stop() throws Exception { - stopCalled = true; - } - - @Override - public void destroy() { - destroyCalled = true; - } - } - - private static class MyContext implements DaemonContext { - - final String[] args; - - MyContext(String... args) { - this.args = args; - } - - @Override - public DaemonController getController() { - return null; - } - - @Override - public String[] getArguments() { - return args; - } - } -} diff --git a/jdisc_core_test/integration_test/pom.xml b/jdisc_core_test/integration_test/pom.xml index 392d1105716..670d812c9e9 100644 --- a/jdisc_core_test/integration_test/pom.xml +++ b/jdisc_core_test/integration_test/pom.xml @@ -301,7 +301,6 @@ </java.util.logging.config.file> <jdisc.bundle.path>${project.build.directory}/dependency</jdisc.bundle.path> <jdisc.cache.path>${project.build.directory}/bundlecache</jdisc.cache.path> - <jdisc.config.file>src/test/resources/config.properties</jdisc.config.file> <jdisc.logger.level>ALL</jdisc.logger.level> </systemPropertyVariables> </configuration> diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/BootstrapDaemonIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/BootstrapDaemonIntegrationTest.java deleted file mode 100644 index d052d2d4715..00000000000 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/BootstrapDaemonIntegrationTest.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.core; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import com.yahoo.jdisc.application.Application; -import org.apache.commons.daemon.DaemonContext; -import org.junit.Test; -import org.mockito.Mockito; - -import static org.junit.Assert.assertEquals; - - -/** - * @author Simon Thoresen Hult - */ -public class BootstrapDaemonIntegrationTest { - - @Test - public void requireThatConfigFileIsInjected() throws Exception { - BootstrapDaemon daemon = new BootstrapDaemon(); - - DaemonContext ctx = Mockito.mock(DaemonContext.class); - Mockito.doReturn(new String[] { MyApplication.class.getName() }).when(ctx).getArguments(); - daemon.init(ctx); - daemon.start(); - - assertEquals("bar", ((MyApplication)((ApplicationLoader)daemon.loader()).application()).foo); - - daemon.stop(); - daemon.destroy(); - } - - public static class MyApplication implements Application { - - final String foo; - - @Inject - public MyApplication(@Named("foo") String foo) { - this.foo = foo; - } - - @Override - public void start() { - - } - - @Override - public void stop() { - - } - - @Override - public void destroy() { - - } - } -} diff --git a/jdisc_core_test/integration_test/src/test/resources/config.properties b/jdisc_core_test/integration_test/src/test/resources/config.properties deleted file mode 100644 index 74d0a43fccf..00000000000 --- a/jdisc_core_test/integration_test/src/test/resources/config.properties +++ /dev/null @@ -1 +0,0 @@ -foo=bar diff --git a/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java b/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java index 459ee60c740..29588c755e4 100644 --- a/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java +++ b/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java @@ -133,8 +133,6 @@ public class CertificateK { private final javax.xml.xpath.XPath xPath = null; private final org.aopalliance.intercept.Joinpoint jointpoint = null; private final org.aopalliance.aop.Advice advice = null; - private final org.apache.commons.daemon.Daemon daemon = null; - private final org.apache.commons.daemon.support.DaemonLoader daemonLoader = null; private final org.apache.commons.logging.LogFactory logFactory = null; private final org.apache.commons.logging.impl.SimpleLog simpleLog = null; private final org.apache.log4j.Appender appender = null; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 865bcc61837..058317ffd25 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.UncheckedIOException; @@ -68,6 +69,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; private final CsrGenerator csrGenerator; + private final boolean useInternalZts; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); @@ -76,7 +78,8 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Path trustStorePath, ConfigServerInfo configServerInfo, String certificateDnsSuffix, - ServiceIdentityProvider hostIdentityProvider) { + ServiceIdentityProvider hostIdentityProvider, + boolean useInternalZts) { this.ztsEndpoint = ztsEndpoint; this.trustStorePath = trustStorePath; this.configserverIdentity = configServerInfo.getConfigServerIdentity(); @@ -87,6 +90,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { hostIdentityProvider, new AthenzIdentityVerifier(singleton(configserverIdentity))); this.clock = Clock.systemUTC(); + this.useInternalZts = useInternalZts; } public boolean converge(NodeAgentContext context) { @@ -157,7 +161,12 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { SignedIdentityDocument signedIdentityDocument = identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( context.identity(), signedIdentityDocument.providerUniqueId(), signedIdentityDocument.ipAddresses(), keyPair); - try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, hostIdentityProvider)) { + + // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis + HostnameVerifier ztsHostNameVerifier = useInternalZts + ? new AthenzIdentityVerifier(singleton(configserverIdentity)) + : null; + try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, hostIdentityProvider, ztsHostNameVerifier)) { InstanceIdentity instanceIdentity = ztsClient.registerInstance( configserverIdentity, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 02161caead6..29c0544420a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -168,7 +168,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { loadBalancerExpirerInterval = Duration.ofMinutes(10); reservationExpiry = Duration.ofMinutes(20); // Need to be long enough for deployment to be finished for all config model versions dynamicProvisionerInterval = Duration.ofMinutes(5); - osUpgradeActivatorInterval = Duration.ofMinutes(5); + osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index 9e36e19f5be..cc6b0d952a3 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -213,8 +213,11 @@ DocsumContext::ParseLocation(search::docsummary::GetDocsumsState *state) } std::unique_ptr<MatchingElements> -DocsumContext::fill_matching_elements(const StructFieldMapper &) +DocsumContext::fill_matching_elements(const StructFieldMapper &struct_field_mapper) { + if (_matcher) { + return _matcher->get_matching_elements(_request, _searchCtx, _attrCtx, _sessionMgr, struct_field_mapper); + } return std::make_unique<MatchingElements>(); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp index 34310371755..71b1164d127 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp @@ -4,6 +4,7 @@ #include <vespa/eval/eval/tensor.h> #include <vespa/eval/eval/tensor_engine.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/searchcommon/attribute/i_search_context.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> #include <vespa/searchlib/queryeval/same_element_blueprint.h> @@ -23,6 +24,8 @@ using search::queryeval::IntermediateBlueprint; using search::queryeval::SameElementBlueprint; using search::queryeval::SearchIterator; +using AttrSearchCtx = search::attribute::ISearchContext; + namespace proton::matching { namespace { @@ -98,11 +101,29 @@ void find_matching_elements(const std::vector<uint32_t> &docs, const SameElement } } +void find_matching_elements(const std::vector<uint32_t> &docs, const vespalib::string &struct_field_name, const AttrSearchCtx &attr_ctx, MatchingElements &result) { + int32_t weight = 0; + std::vector<uint32_t> matches; + for (uint32_t i = 0; i < docs.size(); ++i) { + for (int32_t id = attr_ctx.find(docs[i], 0, weight); id >= 0; id = attr_ctx.find(docs[i], id+1, weight)) { + matches.push_back(id); + } + if (!matches.empty()) { + result.add_matching_elements(docs[i], struct_field_name, matches); + matches.clear(); + } + } +} + void find_matching_elements(const StructFieldMapper &mapper, const std::vector<uint32_t> &docs, const Blueprint &bp, MatchingElements &result) { if (auto same_element = as<SameElementBlueprint>(bp)) { if (mapper.is_struct_field(same_element->struct_field_name())) { find_matching_elements(docs, *same_element, result); } + } else if (const AttrSearchCtx *attr_ctx = bp.get_attribute_search_context()) { + if (mapper.is_struct_subfield(attr_ctx->attributeName())) { + find_matching_elements(docs, mapper.get_struct_field(attr_ctx->attributeName()), *attr_ctx, result); + } } else if (auto and_not = as<AndNotBlueprint>(bp)) { find_matching_elements(mapper, docs, and_not->getChild(0), result); } else if (auto intermediate = as<IntermediateBlueprint>(bp)) { diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 63c1b320fb8..35e21d133c2 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -289,7 +289,8 @@ TEST("testTruncatedIdxFile"){ } const char * magic = "mumbo jumbo"; { - truncate("bug-7257706-truncated/1422358701368384000.idx", 3830); + int truncate_result = truncate("bug-7257706-truncated/1422358701368384000.idx", 3830); + EXPECT_EQUAL(0, truncate_result); LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(), TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); EXPECT_EQUAL(331ul, datastore.lastSyncToken()); diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp index df868de3a97..b7478da3f71 100644 --- a/searchlib/src/tests/fef/properties/properties_test.cpp +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -226,6 +226,14 @@ TEST("test stuff") { EXPECT_TRUE(!eval::LazyExpressions::check(p, true)); EXPECT_TRUE(!eval::LazyExpressions::check(p, false)); } + { // vespa.eval.use_fast_forest + EXPECT_EQUAL(eval::UseFastForest::NAME, vespalib::string("vespa.eval.use_fast_forest")); + EXPECT_EQUAL(eval::UseFastForest::DEFAULT_VALUE, false); + Properties p; + EXPECT_EQUAL(eval::UseFastForest::check(p), false); + p.add("vespa.eval.use_fast_forest", "true"); + EXPECT_EQUAL(eval::UseFastForest::check(p), true); + } { // vespa.rank.firstphase EXPECT_EQUAL(rank::FirstPhase::NAME, vespalib::string("vespa.rank.firstphase")); EXPECT_EQUAL(rank::FirstPhase::DEFAULT_VALUE, vespalib::string("nativeRank")); diff --git a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp index 7e28178e5f7..d1b0f8112f3 100644 --- a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp +++ b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp @@ -90,6 +90,10 @@ struct Fixture { value ? "true" : "false"); return *this; } + Fixture &use_fast_forest() { + indexEnv.getProperties().add(indexproperties::eval::UseFastForest::NAME, "true"); + return *this; + } Fixture &add_expr(const vespalib::string &name, const vespalib::string &expr) { vespalib::string feature_name = expr_feature(name); vespalib::string expr_name = feature_name + ".rankingScript"; @@ -113,6 +117,11 @@ struct Fixture { program.setup(*match_data, queryEnv, overrides); return *this; } + vespalib::string final_executor_name() const { + size_t n = program.num_executors(); + ASSERT_TRUE(n > 0); + return program.get_executor(n-1).getClassName(); + } double get(uint32_t docid = default_docid) { auto result = program.get_seeds(); EXPECT_EQUAL(1u, result.num_features()); @@ -360,4 +369,26 @@ TEST_F("require that interpreted ranking expressions are pure", Fixture()) { EXPECT_EQUAL(f1.get(), 7.0); } +const vespalib::string tree_expr = "if(value(1)<2,1,2)+if(value(2)<1,10,20)"; + +TEST_F("require that fast-forest gbdt evaluation can be enabled", Fixture()) { + f1.use_fast_forest().add_expr("rank", tree_expr).compile(); + EXPECT_EQUAL(f1.get(), 21.0); + EXPECT_EQUAL(f1.final_executor_name(), "search::features::FastForestExecutor"); +} + +TEST_F("require that fast-forest gbdt evaluation is disabled by default", Fixture()) { + f1.add_expr("rank", tree_expr).compile(); + EXPECT_EQUAL(f1.get(), 21.0); + EXPECT_EQUAL(f1.final_executor_name(), "search::features::CompiledRankingExpressionExecutor"); +} + +TEST_F("require that fast-forest gbdt evaluation is pure", Fixture()) { + f1.use_fast_forest().add_expr("rank", tree_expr).compile(); + EXPECT_EQUAL(3u, count_features(f1.program)); + EXPECT_EQUAL(3u, count_const_features(f1.program)); + EXPECT_EQUAL(f1.get(), 21.0); + EXPECT_EQUAL(f1.final_executor_name(), "search::features::FastForestExecutor"); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 192d498125c..5261f568673 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -136,6 +136,10 @@ public: } void visitMembers(vespalib::ObjectVisitor &visitor) const override; + + const attribute::ISearchContext *get_attribute_search_context() const override { + return _search_context.get(); + } }; void diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index 2733ec62105..a4b2280fa57 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -11,20 +11,21 @@ #include <vespa/log/log.h> LOG_SETUP(".features.rankingexpression"); -using vespalib::eval::Function; -using vespalib::eval::PassParams; +using search::fef::FeatureType; +using vespalib::ArrayRef; +using vespalib::ConstArrayRef; using vespalib::eval::CompileCache; using vespalib::eval::CompiledFunction; +using vespalib::eval::DoubleValue; +using vespalib::eval::Function; using vespalib::eval::InterpretedFunction; using vespalib::eval::LazyParams; -using vespalib::eval::ValueType; -using vespalib::eval::Value; -using vespalib::eval::DoubleValue; using vespalib::eval::NodeTypes; +using vespalib::eval::PassParams; +using vespalib::eval::Value; +using vespalib::eval::ValueType; +using vespalib::eval::gbdt::FastForest; using vespalib::tensor::DefaultTensorEngine; -using search::fef::FeatureType; -using vespalib::ArrayRef; -using vespalib::ConstArrayRef; namespace search::features { @@ -43,6 +44,23 @@ vespalib::string list_issues(const std::vector<vespalib::string> &issues) { //----------------------------------------------------------------------------- /** + * Implements the executor for fast forest gbdt evaluation + **/ +class FastForestExecutor : public fef::FeatureExecutor +{ +private: + const FastForest &_forest; + FastForest::Context _ctx; + +public: + FastForestExecutor(const FastForest &forest); + bool isPure() override { return true; } + void execute(uint32_t docId) override; +}; + +//----------------------------------------------------------------------------- + +/** * Implements the executor for compiled ranking expressions **/ class CompiledRankingExpressionExecutor : public fef::FeatureExecutor @@ -110,6 +128,22 @@ public: //----------------------------------------------------------------------------- +FastForestExecutor::FastForestExecutor(const FastForest &forest) + : _forest(forest), + _ctx(_forest) +{ +} + +void +FastForestExecutor::execute(uint32_t) +{ + const auto ¶ms = inputs(); + double result = _forest.eval(_ctx, [¶ms](size_t p){ return params.get_number(p); }); + outputs().set_number(0, result); +} + +//----------------------------------------------------------------------------- + CompiledRankingExpressionExecutor::CompiledRankingExpressionExecutor(const CompiledFunction &compiled_function) : _ranking_function(compiled_function.get_function()), _params(compiled_function.num_params(), 0.0) @@ -178,6 +212,7 @@ RankingExpressionBlueprint::RankingExpressionBlueprint(rankingexpression::Expres : fef::Blueprint("rankingExpression"), _expression_replacer(std::move(replacer)), _intrinsic_expression(), + _fast_forest(), _interpreted_function(), _compile_token(), _input_is_object() @@ -259,11 +294,17 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env, // avoid costly compilation when only verifying setup if (env.getFeatureMotivation() != env.FeatureMotivation::VERIFY_SETUP) { if (do_compile) { - bool suggest_lazy = CompiledFunction::should_use_lazy_params(rank_function); - if (fef::indexproperties::eval::LazyExpressions::check(env.getProperties(), suggest_lazy)) { - _compile_token = CompileCache::compile(rank_function, PassParams::LAZY); - } else { - _compile_token = CompileCache::compile(rank_function, PassParams::ARRAY); + // fast forest evaluation is a possible replacement for compiled tree models + if (fef::indexproperties::eval::UseFastForest::check(env.getProperties())) { + _fast_forest = FastForest::try_convert(rank_function); + } + if (!_fast_forest) { + bool suggest_lazy = CompiledFunction::should_use_lazy_params(rank_function); + if (fef::indexproperties::eval::LazyExpressions::check(env.getProperties(), suggest_lazy)) { + _compile_token = CompileCache::compile(rank_function, PassParams::LAZY); + } else { + _compile_token = CompileCache::compile(rank_function, PassParams::ARRAY); + } } } else { _interpreted_function.reset(new InterpretedFunction(DefaultTensorEngine::ref(), rank_function, node_types)); @@ -300,6 +341,9 @@ RankingExpressionBlueprint::createExecutor(const fef::IQueryEnvironment &env, ve ConstArrayRef<char> input_is_object = stash.copy_array<char>(_input_is_object); return stash.create<InterpretedRankingExpressionExecutor>(*_interpreted_function, input_is_object); } + if (_fast_forest) { + return stash.create<FastForestExecutor>(*_fast_forest); + } assert(_compile_token.get() != nullptr); // will be nullptr for VERIFY_SETUP feature motivation if (_compile_token->get().pass_params() == PassParams::ARRAY) { return stash.create<CompiledRankingExpressionExecutor>(_compile_token->get()); diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h index 104e8d63a70..579c8cf91a7 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/searchlib/fef/blueprint.h> +#include <vespa/eval/eval/fast_forest.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/searchlib/features/rankingexpression/expression_replacer.h> @@ -19,6 +20,7 @@ class RankingExpressionBlueprint : public fef::Blueprint private: rankingexpression::ExpressionReplacer::SP _expression_replacer; rankingexpression::IntrinsicExpression::UP _intrinsic_expression; + vespalib::eval::gbdt::FastForest::UP _fast_forest; vespalib::eval::InterpretedFunction::UP _interpreted_function; vespalib::eval::CompileCache::Token::UP _compile_token; std::vector<char> _input_is_object; diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index a7df39faf2f..ce1bd69cc4c 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -84,6 +84,10 @@ LazyExpressions::check(const Properties &props, bool default_value) return lookupBool(props, NAME, default_value); } +const vespalib::string UseFastForest::NAME("vespa.eval.use_fast_forest"); +const bool UseFastForest::DEFAULT_VALUE(false); +bool UseFastForest::check(const Properties &props) { return lookupBool(props, NAME, DEFAULT_VALUE); } + } // namespace eval namespace rank { diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index 9adf4487ec5..57aa24222a3 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -26,6 +26,13 @@ struct LazyExpressions { static bool check(const Properties &props, bool default_value); }; +// use fast-forest evaluation for gbdt expressions. affects rank/summary/dump +struct UseFastForest { + static const vespalib::string NAME; + static const bool DEFAULT_VALUE; + static bool check(const Properties &props); +}; + } // namespace eval namespace rank { diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.h b/searchlib/src/vespa/searchlib/fef/rank_program.h index 3a92fc874a4..e1014df5ee5 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.h +++ b/searchlib/src/vespa/searchlib/fef/rank_program.h @@ -59,6 +59,7 @@ public: ~RankProgram(); size_t num_executors() const { return _executors.size(); } + const FeatureExecutor &get_executor(size_t i) const { return *_executors[i]; } /** * Set up this rank program by creating the needed feature diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 2f9dbabe52e..907ea9bb066 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -14,6 +14,7 @@ namespace vespalib::slime { struct Cursor; struct Inserter; } +namespace search::attribute { class ISearchContext; } namespace search::queryeval { @@ -198,6 +199,7 @@ public: virtual bool isEquiv() const { return false; } virtual bool isWhiteList() const { return false; } virtual bool isIntermediate() const { return false; } + virtual const attribute::ISearchContext *get_attribute_search_context() const { return nullptr; } }; namespace blueprint { diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 4df636e0219..2a23dd4c495 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -25,6 +25,7 @@ vespa_define_module( src/tests/docsumformat src/tests/docsummary src/tests/docsummary/attribute_combiner + src/tests/docsummary/matched_elements_filter src/tests/docsummary/slime_summary src/tests/extractkeywords ) diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt b/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt new file mode 100644 index 00000000000..a87f5638acc --- /dev/null +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) +vespa_add_executable(searchsummary_matched_elements_filter_test_app TEST + SOURCES + matched_elements_filter_test.cpp + DEPENDS + searchsummary + GTest::GTest +) +vespa_add_test(NAME searchsummary_matched_elements_filter_test_app COMMAND searchsummary_matched_elements_filter_test_app) diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp new file mode 100644 index 00000000000..40d0285b1ec --- /dev/null +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -0,0 +1,205 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/datatype/structdatatype.h> +#include <vespa/document/document.h> +#include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> +#include <vespa/searchsummary/docsummary/docsumstate.h> +#include <vespa/searchsummary/docsummary/idocsumenvironment.h> +#include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> +#include <vespa/searchsummary/docsummary/resultconfig.h> +#include <vespa/searchsummary/docsummary/resultpacker.h> +#include <vespa/searchsummary/docsummary/summaryfieldconverter.h> +#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <iostream> + +#include <vespa/log/log.h> +LOG_SETUP("matched_elements_filter_test"); + +using search::MatchingElements; +using search::StructFieldMapper; +using vespalib::Slime; + +using namespace document; +using namespace search::docsummary; +using namespace vespalib::slime; + +using ElementVector = std::vector<uint32_t>; + +struct SlimeValue { + Slime slime; + + SlimeValue(const std::string& json_input) + : slime() + { + size_t used = JsonFormat::decode(json_input, slime); + EXPECT_GT(used, 0); + } + SlimeValue(const Slime& slime_with_raw_field) + : slime() + { + size_t used = BinaryFormat::decode(slime_with_raw_field.get().asString(), slime); + EXPECT_GT(used, 0); + } +}; + +StructDataType::UP +make_struct_elem_type() +{ + auto result = std::make_unique<StructDataType>("elem"); + result->addField(Field("name", *DataType::STRING, true)); + result->addField(Field("weight", *DataType::INT, true)); + return result; +} + +constexpr uint32_t class_id = 3; +constexpr uint32_t doc_id = 2; + +class DocsumStore { +private: + ResultConfig _config; + ResultPacker _packer; + StructDataType::UP _elem_type; + ArrayDataType _array_type; + MapDataType _map_type; + + StructFieldValue::UP make_elem_value(const std::string& name, int weight) const { + auto result = std::make_unique<StructFieldValue>(*_elem_type); + result->setValue("name", StringFieldValue(name)); + result->setValue("weight", IntFieldValue(weight)); + return result; + } + + void write_field_value(const FieldValue& value) { + auto converted = SummaryFieldConverter::convertSummaryField(false, value); + const auto* raw_field = dynamic_cast<const RawFieldValue*>(converted.get()); + ASSERT_TRUE(raw_field); + auto raw_buf = raw_field->getAsRaw(); + bool result = _packer.AddLongString(raw_buf.first, raw_buf.second); + ASSERT_TRUE(result); + } + +public: + DocsumStore() + : _config(), + _packer(&_config), + _elem_type(make_struct_elem_type()), + _array_type(*_elem_type), + _map_type(*DataType::STRING, *_elem_type) + { + auto* result_class = _config.AddResultClass("test", class_id); + EXPECT_TRUE(result_class->AddConfigEntry("array", ResType::RES_JSONSTRING)); + EXPECT_TRUE(result_class->AddConfigEntry("map", ResType::RES_JSONSTRING)); + _config.CreateEnumMaps(); + } + const ResultConfig& get_config() const { return _config; } + const ResultClass* get_class() const { return _config.LookupResultClass(class_id); } + search::docsummary::DocsumStoreValue getMappedDocsum() { + assert(_packer.Init(class_id)); + { + ArrayFieldValue array_value(_array_type); + array_value.append(make_elem_value("a", 3)); + array_value.append(make_elem_value("b", 5)); + array_value.append(make_elem_value("c", 7)); + write_field_value(array_value); + } + { + MapFieldValue map_value(_map_type); + map_value.put(StringFieldValue("a"), *make_elem_value("a", 3)); + map_value.put(StringFieldValue("b"), *make_elem_value("b", 5)); + map_value.put(StringFieldValue("c"), *make_elem_value("c", 7)); + write_field_value(map_value); + } + const char* buf; + uint32_t buf_len; + assert(_packer.GetDocsumBlob(&buf, &buf_len)); + return DocsumStoreValue(buf, buf_len); + } +}; + +class StateCallback : public GetDocsumsStateCallback { +private: + std::string _field_name; + ElementVector _matching_elements; + +public: + StateCallback(const std::string& field_name, const ElementVector& matching_elements) + : _field_name(field_name), + _matching_elements(matching_elements) + { + } + ~StateCallback() {} + void FillSummaryFeatures(GetDocsumsState*, IDocsumEnvironment*) override {} + void FillRankFeatures(GetDocsumsState*, IDocsumEnvironment*) override {} + void ParseLocation(GetDocsumsState*) override {} + std::unique_ptr<MatchingElements> fill_matching_elements(const StructFieldMapper&) override { + auto result = std::make_unique<MatchingElements>(); + result->add_matching_elements(doc_id, _field_name, _matching_elements); + return result; + } +}; + +class MatchedElementsFilterTest : public ::testing::Test { +private: + DocsumStore _store; + + SlimeValue run_filter_field_writer(const std::string& input_field_name, const ElementVector& matching_elements) { + int input_field_enum = _store.get_config().GetFieldNameEnum().Lookup(input_field_name.c_str()); + EXPECT_GE(input_field_enum, 0); + MatchedElementsFilterDFW filter(input_field_name, input_field_enum); + + GeneralResult result(_store.get_class()); + result.inplaceUnpack(_store.getMappedDocsum()); + StateCallback callback(input_field_name, matching_elements); + GetDocsumsState state(callback); + Slime slime; + SlimeInserter inserter(slime); + + filter.insertField(doc_id, &result, &state, ResType::RES_JSONSTRING, inserter); + return SlimeValue(slime); + } + +public: + MatchedElementsFilterTest() + : _store() + { + } + void expect_filtered(const std::string& input_field_name, const ElementVector& matching_elements, const std::string& exp_slime_as_json) { + SlimeValue act = run_filter_field_writer(input_field_name, matching_elements); + SlimeValue exp(exp_slime_as_json); + EXPECT_EQ(exp.slime, act.slime); + } +}; + +TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value) +{ + expect_filtered("array", {}, "[]"); + expect_filtered("array", {0}, "[{'name':'a','weight':3}]"); + expect_filtered("array", {1}, "[{'name':'b','weight':5}]"); + expect_filtered("array", {2}, "[{'name':'c','weight':7}]"); + expect_filtered("array", {0, 1, 2}, "[{'name':'a','weight':3}," + "{'name':'b','weight':5}," + "{'name':'c','weight':7}]"); +} + +TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value) +{ + expect_filtered("map", {}, "[]"); + expect_filtered("map", {0}, "[{'key':'a','value':{'name':'a','weight':3}}]"); + expect_filtered("map", {1}, "[{'key':'b','value':{'name':'b','weight':5}}]"); + expect_filtered("map", {2}, "[{'key':'c','value':{'name':'c','weight':7}}]"); + expect_filtered("map", {0, 1, 2}, "[{'key':'a','value':{'name':'a','weight':3}}," + "{'key':'b','value':{'name':'b','weight':5}}," + "{'key':'c','value':{'name':'c','weight':7}}]"); +} + +TEST_F(MatchedElementsFilterTest, field_writer_is_not_generated_as_it_depends_on_data_from_document_store) +{ + MatchedElementsFilterDFW filter("foo", 0); + EXPECT_FALSE(filter.IsGenerated()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index dccf72b2fe7..fb6a399e71c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -4,29 +4,30 @@ vespa_add_library(searchsummary_docsummary OBJECT array_attribute_combiner_dfw.cpp attribute_combiner_dfw.cpp attribute_field_writer.cpp - resultclass.cpp - resultconfig.cpp - resultpacker.cpp - urlresult.cpp - getdocsumargs.cpp - docsumstate.cpp + attributedfw.cpp + docsumconfig.cpp docsumfieldwriter.cpp + docsumstate.cpp docsumwriter.cpp - keywordextractor.cpp - attributedfw.cpp dynamicteaserdfw.cpp - docsumconfig.cpp - rankfeaturesdfw.cpp - summaryfeaturesdfw.cpp - juniperproperties.cpp - textextractordfw.cpp geoposdfw.cpp - tokenizer.cpp - positionsdfw.cpp + getdocsumargs.cpp + juniperproperties.cpp + keywordextractor.cpp linguisticsannotation.cpp + matched_elements_filter_dfw.cpp + positionsdfw.cpp + rankfeaturesdfw.cpp + resultclass.cpp + resultconfig.cpp + resultpacker.cpp searchdatatype.cpp struct_map_attribute_combiner_dfw.cpp + summaryfeaturesdfw.cpp summaryfieldconverter.cpp + textextractordfw.cpp + tokenizer.cpp + urlresult.cpp AFTER searchsummary_config ) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp new file mode 100644 index 00000000000..1b7533b53e3 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -0,0 +1,87 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "docsumstate.h" +#include "matched_elements_filter_dfw.h" +#include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/vespalib/data/slime/binary_format.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/smart_buffer.h> +#include <cassert> + +using vespalib::Slime; +using vespalib::slime::ArrayInserter; +using vespalib::slime::BinaryFormat; +using vespalib::slime::Inserter; +using vespalib::slime::Inspector; +using vespalib::slime::SlimeInserter; +using vespalib::slime::inject; + +namespace search::docsummary { + +MatchedElementsFilterDFW::MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum) + : _input_field_name(input_field_name), + _input_field_enum(input_field_enum), + _struct_field_mapper(std::make_shared<StructFieldMapper>()) +{ + // TODO: Take struct field mapper in constructor and populate based on available attribute vectors. +} + +MatchedElementsFilterDFW::~MatchedElementsFilterDFW() = default; + +namespace { + +void +decode_input_field(const ResEntry& entry, search::RawBuf& target_buf, Slime& input_field) +{ + const char* buf; + uint32_t buf_len; + entry._resolve_field(&buf, &buf_len, &target_buf); + BinaryFormat::decode(vespalib::Memory(buf, buf_len), input_field); +} + +void +filter_matching_elements_in_input_field(const Slime& input_field, const std::vector<uint32_t>& matching_elems, Slime& output_field) +{ + SlimeInserter output_inserter(output_field); + Inspector& input_inspector = input_field.get(); + ArrayInserter array_inserter(output_inserter.insertArray()); + auto elems_itr = matching_elems.begin(); + for (size_t i = 0; (i < input_inspector.entries()) && (elems_itr != matching_elems.end()); ++i) { + assert(*elems_itr >= i); + if (*elems_itr == i) { + inject(input_inspector[i], array_inserter); + ++elems_itr; + } + } +} + +void +encode_output_field(const Slime& output_field, Inserter& target) +{ + vespalib::SmartBuffer buf(4096); + BinaryFormat::encode(output_field, buf); + target.insertString(buf.obtain()); +} + +} + +void +MatchedElementsFilterDFW::insertField(uint32_t docid, GeneralResult* result, GetDocsumsState *state, + ResType type, vespalib::slime::Inserter& target) +{ + assert(type == ResType::RES_JSONSTRING); + int entry_idx = result->GetClass()->GetIndexFromEnumValue(_input_field_enum); + ResEntry* entry = result->GetEntry(entry_idx); + if (entry != nullptr) { + Slime input_field; + decode_input_field(*entry, state->_docSumFieldSpace, input_field); + + Slime output_field; + filter_matching_elements_in_input_field(input_field, state->get_matching_elements(*_struct_field_mapper).get_matching_elements(docid, _input_field_name), output_field); + + encode_output_field(output_field, target); + } +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h new file mode 100644 index 00000000000..b96d3595b0a --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h @@ -0,0 +1,27 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsumfieldwriter.h" + +namespace search::docsummary { + +/** + * Field writer that filters matched elements (according to the query) from a complex field + * (map of primitives, map of struct, array of struct) that is retrieved from the document store. + */ +class MatchedElementsFilterDFW : public IDocsumFieldWriter { +private: + std::string _input_field_name; + uint32_t _input_field_enum; + std::shared_ptr<StructFieldMapper> _struct_field_mapper; + +public: + MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum); + ~MatchedElementsFilterDFW(); + bool IsGenerated() const override { return false; } + void insertField(uint32_t docid, GeneralResult* result, GetDocsumsState *state, + ResType type, vespalib::slime::Inserter& target) override; +}; + +} diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index 321b0cc3bba..8409bd60986 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -2,11 +2,10 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/distributor/bucketdbupdater.h> +#include <vespa/storage/distributor/bucket_space_distribution_context.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/pending_bucket_space_db_transition.h> #include <vespa/storage/distributor/outdated_nodes_map.h> -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storage/storageutil/distributorstatecache.h> #include <tests/distributor/distributortestutil.h> #include <vespa/document/test/make_document_bucket.h> @@ -124,7 +123,7 @@ public: createLinks(); _bucketSpaces = getBucketSpaces(); // Disable deferred activation by default (at least for now) to avoid breaking the entire world. - getConfig().setAllowStaleReadsDuringClusterStateTransitions(false); + getBucketDBUpdater().set_stale_reads_enabled(false); }; void TearDown() override { @@ -2415,7 +2414,7 @@ void for_each_bucket(const DistributorBucketSpaceRepo& repo, Func&& f) { } TEST_F(BucketDBUpdaterTest, non_owned_buckets_moved_to_read_only_db_on_ownership_change) { - getConfig().setAllowStaleReadsDuringClusterStateTransitions(true); + getBucketDBUpdater().set_stale_reads_enabled(true); lib::ClusterState initial_state("distributor:1 storage:4"); // All buckets owned by us by definition set_cluster_state_bundle(lib::ClusterStateBundle(initial_state, {}, false)); // Skip activation step for simplicity @@ -2468,7 +2467,7 @@ TEST_F(BucketDBUpdaterTest, buckets_no_longer_available_are_not_moved_to_read_on } TEST_F(BucketDBUpdaterTest, non_owned_buckets_purged_when_read_only_support_is_config_disabled) { - getConfig().setAllowStaleReadsDuringClusterStateTransitions(false); + getBucketDBUpdater().set_stale_reads_enabled(false); lib::ClusterState initial_state("distributor:1 storage:4"); // All buckets owned by us by definition set_cluster_state_bundle(lib::ClusterStateBundle(initial_state, {}, false)); // Skip activation step for simplicity @@ -2497,7 +2496,6 @@ void BucketDBUpdaterTest::trigger_completed_but_not_yet_activated_transition( uint32_t pending_buckets, uint32_t pending_expected_msgs) { - getConfig().setAllowStaleReadsDuringClusterStateTransitions(true); lib::ClusterState initial_state(initial_state_str); setSystemState(initial_state); ASSERT_EQ(messageCount(initial_expected_msgs), _sender.commands().size()); @@ -2514,6 +2512,7 @@ void BucketDBUpdaterTest::trigger_completed_but_not_yet_activated_transition( } TEST_F(BucketDBUpdaterTest, deferred_activated_state_does_not_enable_state_until_activation_received) { + getBucketDBUpdater().set_stale_reads_enabled(true); constexpr uint32_t n_buckets = 10; ASSERT_NO_FATAL_FAILURE( trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4, @@ -2533,6 +2532,7 @@ TEST_F(BucketDBUpdaterTest, deferred_activated_state_does_not_enable_state_until } TEST_F(BucketDBUpdaterTest, read_only_db_cleared_once_pending_state_is_activated) { + getBucketDBUpdater().set_stale_reads_enabled(true); constexpr uint32_t n_buckets = 10; ASSERT_NO_FATAL_FAILURE( trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, @@ -2544,6 +2544,7 @@ TEST_F(BucketDBUpdaterTest, read_only_db_cleared_once_pending_state_is_activated } TEST_F(BucketDBUpdaterTest, read_only_db_is_populated_even_when_self_is_marked_down) { + getBucketDBUpdater().set_stale_reads_enabled(true); constexpr uint32_t n_buckets = 10; ASSERT_NO_FATAL_FAILURE( trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, @@ -2557,6 +2558,7 @@ TEST_F(BucketDBUpdaterTest, read_only_db_is_populated_even_when_self_is_marked_d } TEST_F(BucketDBUpdaterTest, activate_cluster_state_request_with_mismatching_version_returns_actual_version) { + getBucketDBUpdater().set_stale_reads_enabled(true); constexpr uint32_t n_buckets = 10; ASSERT_NO_FATAL_FAILURE( trigger_completed_but_not_yet_activated_transition("version:4 distributor:1 storage:4", n_buckets, 4, @@ -2570,6 +2572,7 @@ TEST_F(BucketDBUpdaterTest, activate_cluster_state_request_with_mismatching_vers } TEST_F(BucketDBUpdaterTest, activate_cluster_state_request_without_pending_transition_passes_message_through) { + getBucketDBUpdater().set_stale_reads_enabled(true); constexpr uint32_t n_buckets = 10; ASSERT_NO_FATAL_FAILURE( trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4, @@ -2727,4 +2730,136 @@ TEST_F(BucketDBUpdaterTest, pending_cluster_state_getter_is_non_null_only_when_s EXPECT_TRUE(state == nullptr); } +struct BucketDBUpdaterSnapshotTest : BucketDBUpdaterTest { + lib::ClusterState empty_state; + std::shared_ptr<lib::ClusterState> initial_baseline; + std::shared_ptr<lib::ClusterState> initial_default; + lib::ClusterStateBundle initial_bundle; + Bucket default_bucket; + Bucket global_bucket; + + BucketDBUpdaterSnapshotTest() + : BucketDBUpdaterTest(), + empty_state(), + initial_baseline(std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:d")), + initial_default(std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:m")), + initial_bundle(*initial_baseline, {{FixedBucketSpaces::default_space(), initial_default}, + {FixedBucketSpaces::global_space(), initial_baseline}}), + default_bucket(FixedBucketSpaces::default_space(), BucketId(16, 1234)), + global_bucket(FixedBucketSpaces::global_space(), BucketId(16, 1234)) + { + } + ~BucketDBUpdaterSnapshotTest() override; + + void SetUp() override { + BucketDBUpdaterTest::SetUp(); + getBucketDBUpdater().set_stale_reads_enabled(true); + }; + + // Assumes that the distributor owns all buckets, so it may choose any arbitrary bucket in the bucket space + uint32_t buckets_in_snapshot_matching_current_db(DistributorBucketSpaceRepo& repo, BucketSpace bucket_space) { + auto rs = getBucketDBUpdater().read_snapshot_for_bucket(Bucket(bucket_space, BucketId(16, 1234))); + if (!rs.is_routable()) { + return 0; + } + auto guard = rs.steal_read_guard(); + uint32_t found_buckets = 0; + for_each_bucket(repo, [&](const auto& space, const auto& entry) { + if (space == bucket_space) { + std::vector<BucketDatabase::Entry> entries; + guard->find_parents_and_self(entry.getBucketId(), entries); + if (entries.size() == 1) { + ++found_buckets; + } + } + }); + return found_buckets; + } +}; + +BucketDBUpdaterSnapshotTest::~BucketDBUpdaterSnapshotTest() = default; + +TEST_F(BucketDBUpdaterSnapshotTest, default_space_snapshot_prior_to_activated_state_is_non_routable) { + auto rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); + EXPECT_FALSE(rs.is_routable()); +} + +TEST_F(BucketDBUpdaterSnapshotTest, global_space_snapshot_prior_to_activated_state_is_non_routable) { + auto rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket); + EXPECT_FALSE(rs.is_routable()); +} + +TEST_F(BucketDBUpdaterSnapshotTest, read_snapshot_returns_appropriate_cluster_states) { + set_cluster_state_bundle(initial_bundle); + // State currently pending, empty initial state is active + + auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); + EXPECT_EQ(def_rs.context().active_cluster_state()->toString(), empty_state.toString()); + EXPECT_EQ(def_rs.context().default_active_cluster_state()->toString(), empty_state.toString()); + ASSERT_TRUE(def_rs.context().has_pending_state_transition()); + EXPECT_EQ(def_rs.context().pending_cluster_state()->toString(), initial_default->toString()); + + auto global_rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket); + EXPECT_EQ(global_rs.context().active_cluster_state()->toString(), empty_state.toString()); + EXPECT_EQ(global_rs.context().default_active_cluster_state()->toString(), empty_state.toString()); + ASSERT_TRUE(global_rs.context().has_pending_state_transition()); + EXPECT_EQ(global_rs.context().pending_cluster_state()->toString(), initial_baseline->toString()); + + ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(*initial_baseline, messageCount(1), 0)); + // State now activated, no pending + + def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); + EXPECT_EQ(def_rs.context().active_cluster_state()->toString(), initial_default->toString()); + EXPECT_EQ(def_rs.context().default_active_cluster_state()->toString(), initial_default->toString()); + EXPECT_FALSE(def_rs.context().has_pending_state_transition()); + + global_rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket); + EXPECT_EQ(global_rs.context().active_cluster_state()->toString(), initial_baseline->toString()); + EXPECT_EQ(global_rs.context().default_active_cluster_state()->toString(), initial_default->toString()); + EXPECT_FALSE(global_rs.context().has_pending_state_transition()); +} + +TEST_F(BucketDBUpdaterSnapshotTest, snapshot_with_no_pending_state_transition_returns_mutable_db_guard) { + constexpr uint32_t n_buckets = 10; + ASSERT_NO_FATAL_FAILURE( + trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4, + "version:2 distributor:1 storage:4", n_buckets, 4)); + EXPECT_FALSE(activate_cluster_state_version(2)); + EXPECT_EQ(buckets_in_snapshot_matching_current_db(mutable_repo(), FixedBucketSpaces::default_space()), + n_buckets); + EXPECT_EQ(buckets_in_snapshot_matching_current_db(mutable_repo(), FixedBucketSpaces::global_space()), + n_buckets); +} + +TEST_F(BucketDBUpdaterSnapshotTest, snapshot_returns_unroutable_for_non_owned_bucket_in_current_state) { + ASSERT_NO_FATAL_FAILURE( + trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4, + "version:2 distributor:2 .0.s:d storage:4", 0, 0)); + EXPECT_FALSE(activate_cluster_state_version(2)); + // We're down in state 2 and therefore do not own any buckets + auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); + EXPECT_FALSE(def_rs.is_routable()); +} + +TEST_F(BucketDBUpdaterSnapshotTest, snapshot_with_pending_state_returns_read_only_guard_for_bucket_only_owned_in_current_state) { + constexpr uint32_t n_buckets = 10; + ASSERT_NO_FATAL_FAILURE( + trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, + "version:2 distributor:2 .0.s:d storage:4", 0, 0)); + EXPECT_EQ(buckets_in_snapshot_matching_current_db(read_only_repo(), FixedBucketSpaces::default_space()), + n_buckets); + EXPECT_EQ(buckets_in_snapshot_matching_current_db(read_only_repo(), FixedBucketSpaces::global_space()), + n_buckets); +} + +TEST_F(BucketDBUpdaterSnapshotTest, snapshot_is_unroutable_if_stale_reads_disabled_and_bucket_not_owned_in_pending_state) { + getBucketDBUpdater().set_stale_reads_enabled(false); + constexpr uint32_t n_buckets = 10; + ASSERT_NO_FATAL_FAILURE( + trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, + "version:2 distributor:2 .0.s:d storage:4", 0, 0)); + auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); + EXPECT_FALSE(def_rs.is_routable()); +} + } diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp index 4a9ef147741..15820b64ff9 100644 --- a/storage/src/tests/distributor/distributortestutil.cpp +++ b/storage/src/tests/distributor/distributortestutil.cpp @@ -417,7 +417,8 @@ DistributorTestUtil::getBucketSpaces() const void DistributorTestUtil::enableDistributorClusterState(vespalib::stringref state) { - _distributor->enableClusterStateBundle(lib::ClusterStateBundle(lib::ClusterState(state))); + getBucketDBUpdater().simulate_cluster_state_bundle_activation( + lib::ClusterStateBundle(lib::ClusterState(state))); } } diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp index 600e56faf31..84f7d34d069 100644 --- a/storage/src/tests/distributor/externaloperationhandlertest.cpp +++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp @@ -505,6 +505,7 @@ document::BucketId ExternalOperationHandlerTest::set_up_pending_cluster_state_tr std::string current = "version:123 distributor:2 storage:2"; std::string pending = "version:321 distributor:3 storage:3"; setupDistributor(1, 3, current); + getBucketDBUpdater().set_stale_reads_enabled(read_only_enabled); getConfig().setAllowStaleReadsDuringClusterStateTransitions(read_only_enabled); // Trigger pending cluster state diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp index 7c308e152db..99d7c12551d 100644 --- a/storage/src/tests/distributor/getoperationtest.cpp +++ b/storage/src/tests/distributor/getoperationtest.cpp @@ -3,6 +3,8 @@ #include <vespa/config/helper/configgetter.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/externaloperationhandler.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributormetricsset.h> @@ -31,7 +33,7 @@ struct GetOperationTest : Test, DistributorTestUtil { std::unique_ptr<Operation> op; GetOperationTest(); - ~GetOperationTest(); + ~GetOperationTest() override; void SetUp() override { _repo.reset( @@ -53,6 +55,7 @@ struct GetOperationTest : Test, DistributorTestUtil { auto msg = std::make_shared<api::GetCommand>(makeDocumentBucket(document::BucketId(0)), docId, "[all]"); op = std::make_unique<GetOperation>( getExternalOperationHandler(), getDistributorBucketSpace(), + getDistributorBucketSpace().getBucketDatabase().acquire_read_guard(), msg, getDistributor().getMetrics(). gets[msg->getLoadType()]); op->start(_sender, framework::MilliSecTime(0)); } diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp index c3ade3c2877..66d44a655e0 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp @@ -148,7 +148,9 @@ Entry BTreeBucketDatabase::entry_from_iterator(const BTree::ConstIterator& iter) if (!iter.valid()) { return Entry::createInvalid(); } - return entry_from_value(iter.getKey(), iter.getData()); + const auto value = iter.getData(); + std::atomic_thread_fence(std::memory_order_acquire); + return entry_from_value(iter.getKey(), value); } ConstEntryRef BTreeBucketDatabase::const_entry_ref_from_iterator(const BTree::ConstIterator& iter) const { @@ -156,6 +158,7 @@ ConstEntryRef BTreeBucketDatabase::const_entry_ref_from_iterator(const BTree::Co return ConstEntryRef::createInvalid(); } const auto value = iter.getData(); + std::atomic_thread_fence(std::memory_order_acquire); const auto replicas_ref = _store.get(entry_ref_from_value(value)); const auto bucket = BucketId(BucketId::keyToBucketId(iter.getKey())); return const_entry_ref_from_replica_array_ref(bucket, gc_timestamp_from_value(value), replicas_ref); diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index 8c701033e67..944df6e1708 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(storage_distributor bucket_db_prune_elision.cpp bucketgctimecalculator.cpp bucketlistmerger.cpp + bucket_space_distribution_context.cpp clusterinformation.cpp distributor_bucket_space.cpp distributor_bucket_space_repo.cpp @@ -20,6 +21,7 @@ vespa_add_library(storage_distributor idealstatemetricsset.cpp messagetracker.cpp nodeinfo.cpp + operation_routing_snapshot.cpp operation_sequencer.cpp operationowner.cpp operationtargetresolver.cpp diff --git a/storage/src/vespa/storage/distributor/bucket_space_distribution_context.cpp b/storage/src/vespa/storage/distributor/bucket_space_distribution_context.cpp new file mode 100644 index 00000000000..53040bc42b1 --- /dev/null +++ b/storage/src/vespa/storage/distributor/bucket_space_distribution_context.cpp @@ -0,0 +1,81 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "bucket_space_distribution_context.h" + +namespace storage::distributor { + +BucketSpaceDistributionContext::~BucketSpaceDistributionContext() = default; + +BucketSpaceDistributionContext::BucketSpaceDistributionContext( + std::shared_ptr<const lib::ClusterState> active_cluster_state, + std::shared_ptr<const lib::ClusterState> default_active_cluster_state, + std::shared_ptr<const lib::ClusterState> pending_cluster_state, + std::shared_ptr<const lib::Distribution> distribution, + uint16_t this_node_index) + : _active_cluster_state(std::move(active_cluster_state)), + _default_active_cluster_state(std::move(default_active_cluster_state)), + _pending_cluster_state(std::move(pending_cluster_state)), + _distribution(std::move(distribution)), + _this_node_index(this_node_index) +{} + +std::shared_ptr<BucketSpaceDistributionContext> BucketSpaceDistributionContext::make_state_transition( + std::shared_ptr<const lib::ClusterState> active_cluster_state, + std::shared_ptr<const lib::ClusterState> default_active_cluster_state, + std::shared_ptr<const lib::ClusterState> pending_cluster_state, + std::shared_ptr<const lib::Distribution> distribution, + uint16_t this_node_index) +{ + return std::make_shared<BucketSpaceDistributionContext>( + std::move(active_cluster_state), std::move(default_active_cluster_state), + std::move(pending_cluster_state), std::move(distribution), + this_node_index); +} + +std::shared_ptr<BucketSpaceDistributionContext> BucketSpaceDistributionContext::make_stable_state( + std::shared_ptr<const lib::ClusterState> active_cluster_state, + std::shared_ptr<const lib::ClusterState> default_active_cluster_state, + std::shared_ptr<const lib::Distribution> distribution, + uint16_t this_node_index) +{ + return std::make_shared<BucketSpaceDistributionContext>( + std::move(active_cluster_state), std::move(default_active_cluster_state), + std::shared_ptr<const lib::ClusterState>(), + std::move(distribution), this_node_index); +} + +std::shared_ptr<BucketSpaceDistributionContext> +BucketSpaceDistributionContext::make_not_yet_initialized(uint16_t this_node_index) +{ + return std::make_shared<BucketSpaceDistributionContext>( + std::make_shared<const lib::ClusterState>(), + std::make_shared<const lib::ClusterState>(), + std::shared_ptr<const lib::ClusterState>(), + std::make_shared<const lib::Distribution>(), + this_node_index); +} + +bool BucketSpaceDistributionContext::bucket_owned_in_state(const lib::ClusterState& state, + const document::BucketId& id) const +{ + try { + uint16_t owner_idx = _distribution->getIdealDistributorNode(state, id); + return (owner_idx == _this_node_index); + } catch (lib::TooFewBucketBitsInUseException&) { + return false; + } catch (lib::NoDistributorsAvailableException&) { + return false; + } +} + +bool BucketSpaceDistributionContext::bucket_owned_in_active_state(const document::BucketId& id) const { + return bucket_owned_in_state(*_active_cluster_state, id); +} + +bool BucketSpaceDistributionContext::bucket_owned_in_pending_state(const document::BucketId& id) const { + if (_pending_cluster_state) { + return bucket_owned_in_state(*_pending_cluster_state, id); + } + return true; // No pending state, owned by default. +} + +} diff --git a/storage/src/vespa/storage/distributor/bucket_space_distribution_context.h b/storage/src/vespa/storage/distributor/bucket_space_distribution_context.h new file mode 100644 index 00000000000..7a9c0fcae60 --- /dev/null +++ b/storage/src/vespa/storage/distributor/bucket_space_distribution_context.h @@ -0,0 +1,70 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdslib/distribution/distribution.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <memory> +#include <cstdint> + +namespace storage::distributor { + +/** + * Represents a consistent snapshot of cluster state and distribution config + * information at a particular point in time. This is sufficient to compute + * bucket ownership and distributions for the bucket space associated with + * the context. + * + * Since this is a snapshot in time, the context is immutable once created. + */ +class BucketSpaceDistributionContext { + std::shared_ptr<const lib::ClusterState> _active_cluster_state; + std::shared_ptr<const lib::ClusterState> _default_active_cluster_state; + std::shared_ptr<const lib::ClusterState> _pending_cluster_state; // May be null if no state is pending + std::shared_ptr<const lib::Distribution> _distribution; // TODO ideally should have a pending distribution as well + uint16_t _this_node_index; +public: + BucketSpaceDistributionContext() = delete; + // Public due to make_shared, prefer factory functions to instantiate instead. + BucketSpaceDistributionContext(std::shared_ptr<const lib::ClusterState> active_cluster_state, + std::shared_ptr<const lib::ClusterState> default_active_cluster_state, + std::shared_ptr<const lib::ClusterState> pending_cluster_state, + std::shared_ptr<const lib::Distribution> distribution, + uint16_t this_node_index); + ~BucketSpaceDistributionContext(); + + static std::shared_ptr<BucketSpaceDistributionContext> make_state_transition( + std::shared_ptr<const lib::ClusterState> active_cluster_state, + std::shared_ptr<const lib::ClusterState> default_active_cluster_state, + std::shared_ptr<const lib::ClusterState> pending_cluster_state, + std::shared_ptr<const lib::Distribution> distribution, + uint16_t this_node_index); + static std::shared_ptr<BucketSpaceDistributionContext> make_stable_state( + std::shared_ptr<const lib::ClusterState> active_cluster_state, + std::shared_ptr<const lib::ClusterState> default_active_cluster_state, + std::shared_ptr<const lib::Distribution> distribution, + uint16_t this_node_index); + static std::shared_ptr<BucketSpaceDistributionContext> make_not_yet_initialized(uint16_t this_node_index); + + const std::shared_ptr<const lib::ClusterState>& active_cluster_state() const noexcept { + return _active_cluster_state; + } + + const std::shared_ptr<const lib::ClusterState>& default_active_cluster_state() const noexcept { + return _default_active_cluster_state; + } + bool has_pending_state_transition() const noexcept { + return (_pending_cluster_state.get() != nullptr); + } + // Returned shared_ptr is nullptr iff has_pending_state_transition() == false. + const std::shared_ptr<const lib::ClusterState>& pending_cluster_state() const noexcept { + return _pending_cluster_state; + } + + bool bucket_owned_in_state(const lib::ClusterState& state, const document::BucketId& id) const; + bool bucket_owned_in_active_state(const document::BucketId& id) const; + bool bucket_owned_in_pending_state(const document::BucketId& id) const; + + uint16_t this_node_index() const noexcept { return _this_node_index; } +}; + +} diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index a901ac28a54..227165a0911 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -2,6 +2,7 @@ #include "bucketdbupdater.h" #include "bucket_db_prune_elision.h" +#include "bucket_space_distribution_context.h" #include "distributor.h" #include "distributor_bucket_space.h" #include "distributormetricsset.h" @@ -30,12 +31,44 @@ BucketDBUpdater::BucketDBUpdater(Distributor& owner, : framework::StatusReporter("bucketdb", "Bucket DB Updater"), _distributorComponent(owner, bucketSpaceRepo, readOnlyBucketSpaceRepo, compReg, "Bucket DB Updater"), _sender(sender), - _transitionTimer(_distributorComponent.getClock()) + _transitionTimer(_distributorComponent.getClock()), + _active_distribution_contexts(), + _distribution_context_mutex() { + for (auto& elem : _distributorComponent.getBucketSpaceRepo()) { + _active_distribution_contexts.emplace( + elem.first, + BucketSpaceDistributionContext::make_not_yet_initialized(_distributorComponent.getIndex())); + _explicit_transition_read_guard.emplace(elem.first, std::shared_ptr<BucketDatabase::ReadGuard>()); + } } BucketDBUpdater::~BucketDBUpdater() = default; +OperationRoutingSnapshot BucketDBUpdater::read_snapshot_for_bucket(const document::Bucket& bucket) const { + const auto bucket_space = bucket.getBucketSpace(); + std::lock_guard lock(_distribution_context_mutex); + auto active_state_iter = _active_distribution_contexts.find(bucket_space); + assert(active_state_iter != _active_distribution_contexts.cend()); + auto& state = *active_state_iter->second; + if (!state.bucket_owned_in_active_state(bucket.getBucketId())) { + return OperationRoutingSnapshot::make_not_routable_in_state(active_state_iter->second); + } + const bool bucket_present_in_mutable_db = state.bucket_owned_in_pending_state(bucket.getBucketId()); + if (!bucket_present_in_mutable_db && !stale_reads_enabled()) { + return OperationRoutingSnapshot::make_not_routable_in_state(active_state_iter->second); + } + const auto& space_repo = bucket_present_in_mutable_db + ? _distributorComponent.getBucketSpaceRepo() + : _distributorComponent.getReadOnlyBucketSpaceRepo(); + auto existing_guard_iter = _explicit_transition_read_guard.find(bucket_space); + assert(existing_guard_iter != _explicit_transition_read_guard.cend()); + auto db_guard = existing_guard_iter->second + ? existing_guard_iter-> second + : space_repo.get(bucket_space).getBucketDatabase().acquire_read_guard(); + return OperationRoutingSnapshot::make_routable_with_guard(active_state_iter->second, std::move(db_guard), space_repo); +} + void BucketDBUpdater::flush() { @@ -59,8 +92,7 @@ BucketDBUpdater::print(std::ostream& out, bool verbose, const std::string& inden bool BucketDBUpdater::shouldDeferStateEnabling() const noexcept { - return _distributorComponent.getDistributor().getConfig() - .allowStaleReadsDuringClusterStateTransitions(); + return stale_reads_enabled(); } bool @@ -258,6 +290,61 @@ BucketDBUpdater::replyToActivationWithActualVersion( _distributorComponent.sendUp(reply); // TODO let API accept rvalues } +void BucketDBUpdater::update_read_snapshot_before_db_pruning() { + std::lock_guard lock(_distribution_context_mutex); + for (auto& elem : _distributorComponent.getBucketSpaceRepo()) { + // At this point, we're still operating with a distribution context _without_ a + // pending state, i.e. anyone using the context will expect to find buckets + // in the DB that correspond to how the database looked like prior to pruning + // buckets from the DB. To ensure this is not violated, take a snapshot of the + // _mutable_ DB and expose this. This snapshot only lives until we atomically + // flip to expose a distribution context that includes the new, pending state. + // At that point, the read-only DB is known to contain the buckets that have + // been pruned away, so we can release the mutable DB snapshot safely. + // TODO test for, and handle, state preemption case! + _explicit_transition_read_guard[elem.first] = elem.second->getBucketDatabase().acquire_read_guard(); + } +} + + +void BucketDBUpdater::update_read_snapshot_after_db_pruning(const lib::ClusterStateBundle& new_state) { + std::lock_guard lock(_distribution_context_mutex); + const auto old_default_state = _distributorComponent.getBucketSpaceRepo().get( + document::FixedBucketSpaces::default_space()).cluster_state_sp(); + for (auto& elem : _distributorComponent.getBucketSpaceRepo()) { + auto new_distribution = elem.second->distribution_sp(); + auto old_cluster_state = elem.second->cluster_state_sp(); + auto new_cluster_state = new_state.getDerivedClusterState(elem.first); + _active_distribution_contexts.insert_or_assign( + elem.first, + BucketSpaceDistributionContext::make_state_transition( + std::move(old_cluster_state), + old_default_state, + std::move(new_cluster_state), + std::move(new_distribution), + _distributorComponent.getIndex())); + // We can now remove the explicit mutable DB snapshot, as the buckets that have been + // pruned away are visible in the read-only DB. + _explicit_transition_read_guard[elem.first] = std::shared_ptr<BucketDatabase::ReadGuard>(); + } +} + +void BucketDBUpdater::update_read_snapshot_after_activation(const lib::ClusterStateBundle& activated_state) { + std::lock_guard lock(_distribution_context_mutex); + const auto& default_cluster_state = activated_state.getDerivedClusterState(document::FixedBucketSpaces::default_space()); + for (auto& elem : _distributorComponent.getBucketSpaceRepo()) { + auto new_distribution = elem.second->distribution_sp(); + auto new_cluster_state = activated_state.getDerivedClusterState(elem.first); + _active_distribution_contexts.insert_or_assign( + elem.first, + BucketSpaceDistributionContext::make_stable_state( + std::move(new_cluster_state), + default_cluster_state, + std::move(new_distribution), + _distributorComponent.getIndex())); + } +} + bool BucketDBUpdater::onSetSystemState( const std::shared_ptr<api::SetSystemStateCommand>& cmd) @@ -275,8 +362,10 @@ BucketDBUpdater::onSetSystemState( ensureTransitionTimerStarted(); // Separate timer since _transitionTimer might span multiple pending states. framework::MilliSecTimer process_timer(_distributorComponent.getClock()); - - removeSuperfluousBuckets(cmd->getClusterStateBundle(), false); + update_read_snapshot_before_db_pruning(); + const auto& bundle = cmd->getClusterStateBundle(); + removeSuperfluousBuckets(bundle, false); + update_read_snapshot_after_db_pruning(bundle); replyToPreviousPendingClusterStateIfAny(); ClusterInformation::CSP clusterInfo( @@ -642,6 +731,7 @@ BucketDBUpdater::activatePendingClusterState() _distributorComponent.getDistributor().notifyDistributionChangeEnabled(); } + update_read_snapshot_after_activation(_pendingClusterState->getNewClusterStateBundle()); _pendingClusterState.reset(); _outdatedNodesMap.clear(); sendAllQueuedBucketRechecks(); @@ -665,6 +755,11 @@ BucketDBUpdater::enableCurrentClusterStateBundleInDistributor() _distributorComponent.getDistributor().enableClusterStateBundle(state); } +void BucketDBUpdater::simulate_cluster_state_bundle_activation(const lib::ClusterStateBundle& activated_state) { + update_read_snapshot_after_activation(activated_state); + _distributorComponent.getDistributor().enableClusterStateBundle(activated_state); +} + void BucketDBUpdater::addCurrentStateToClusterStateHistory() { diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index e69d328d8bc..86ceab14486 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -6,6 +6,7 @@ #include "distributorcomponent.h" #include "distributormessagesender.h" #include "pendingclusterstate.h" +#include "operation_routing_snapshot.h" #include "outdated_nodes_map.h" #include <vespa/document/bucket/bucket.h> #include <vespa/storageapi/messageapi/returncode.h> @@ -15,7 +16,9 @@ #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageframework/generic/status/statusreporter.h> #include <vespa/storageapi/messageapi/messagehandler.h> +#include <atomic> #include <list> +#include <mutex> namespace vespalib::xml { class XmlOutputStream; @@ -25,6 +28,7 @@ class XmlAttribute; namespace storage::distributor { class Distributor; +class BucketSpaceDistributionContext; class BucketDBUpdater : public framework::StatusReporter, public api::MessageHandler @@ -70,7 +74,14 @@ public: return ((_pendingClusterState.get() != nullptr) && _pendingClusterState->hasBucketOwnershipTransfer()); } + void set_stale_reads_enabled(bool enabled) noexcept { + _stale_reads_enabled.store(enabled, std::memory_order_relaxed); + } + bool stale_reads_enabled() const noexcept { + return _stale_reads_enabled.load(std::memory_order_relaxed); + } + OperationRoutingSnapshot read_snapshot_for_bucket(const document::Bucket&) const; private: DistributorComponent _distributorComponent; class MergeReplyGuard { @@ -129,6 +140,12 @@ private: } }; + friend class DistributorTestUtil; + // Only to be used by tests that want to ensure both the BucketDBUpdater _and_ the Distributor + // components agree on the currently active cluster state bundle. + // Transitively invokes Distributor::enableClusterStateBundle + void simulate_cluster_state_bundle_activation(const lib::ClusterStateBundle& activated_state); + bool shouldDeferStateEnabling() const noexcept; bool hasPendingClusterState() const; bool pendingClusterStateAccepted(const std::shared_ptr<api::RequestBucketInfoReply>& repl); @@ -166,8 +183,11 @@ private: void updateState(const lib::ClusterState& oldState, const lib::ClusterState& newState); + void update_read_snapshot_before_db_pruning(); void removeSuperfluousBuckets(const lib::ClusterStateBundle& newState, bool is_distribution_config_change); + void update_read_snapshot_after_db_pruning(const lib::ClusterStateBundle& new_state); + void update_read_snapshot_after_activation(const lib::ClusterStateBundle& activated_state); void replyToPreviousPendingClusterStateIfAny(); void replyToActivationWithActualVersion( @@ -182,9 +202,6 @@ private: void maybe_inject_simulated_db_pruning_delay(); void maybe_inject_simulated_db_merging_delay(); - friend class BucketDBUpdater_Test; - friend class MergeOperation_Test; - /** Removes all copies of buckets that are on nodes that are down. */ @@ -235,6 +252,16 @@ private: std::set<EnqueuedBucketRecheck> _enqueuedRechecks; OutdatedNodesMap _outdatedNodesMap; framework::MilliSecTimer _transitionTimer; + std::atomic<bool> _stale_reads_enabled; + using DistributionContexts = std::unordered_map<document::BucketSpace, + std::shared_ptr<BucketSpaceDistributionContext>, + document::BucketSpace::hash>; + DistributionContexts _active_distribution_contexts; + using DbGuards = std::unordered_map<document::BucketSpace, + std::shared_ptr<BucketDatabase::ReadGuard>, + document::BucketSpace::hash>; + DbGuards _explicit_transition_read_guard; + mutable std::mutex _distribution_context_mutex; }; } diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 69b64ac8dc1..4adbdd32669 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -77,7 +77,8 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _distributorStatusDelegate(compReg, *this, *this), _bucketDBStatusDelegate(compReg, *this, _bucketDBUpdater), _idealStateManager(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, compReg, manageActiveBucketCopies), - _externalOperationHandler(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, _idealStateManager, compReg), + _externalOperationHandler(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, + _idealStateManager, compReg, use_btree_database), _threadPool(threadPool), _initializingIsUp(true), _doneInitializeHandler(doneInitHandler), @@ -322,9 +323,7 @@ bool Distributor::handleMessage(const std::shared_ptr<api::StorageMessage>& msg) { if (msg->getType().isReply()) { - std::shared_ptr<api::StorageReply> reply = - std::dynamic_pointer_cast<api::StorageReply>(msg); - + auto reply = std::dynamic_pointer_cast<api::StorageReply>(msg); if (handleReply(reply)) { return true; } @@ -400,6 +399,10 @@ Distributor::enableClusterStateBundle(const lib::ClusterStateBundle& state) } } +OperationRoutingSnapshot Distributor::read_snapshot_for_bucket(const document::Bucket& bucket) const { + return _bucketDBUpdater.read_snapshot_for_bucket(bucket); +} + void Distributor::notifyDistributionChangeEnabled() { @@ -834,6 +837,7 @@ Distributor::enableNextConfig() _bucketDBMetricUpdater.setMinimumReplicaCountingMode(getConfig().getMinimumReplicaCountingMode()); _ownershipSafeTimeCalc->setMaxClusterClockSkew(getConfig().getMaxClusterClockSkew()); _pendingMessageTracker.setNodeBusyDuration(getConfig().getInhibitMergesOnBusyNodeDuration()); + _bucketDBUpdater.set_stale_reads_enabled(getConfig().allowStaleReadsDuringClusterStateTransitions()); } void diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 638704adf24..48d9145eec7 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -170,6 +170,8 @@ public: return *_readOnlyBucketSpaceRepo; } + OperationRoutingSnapshot read_snapshot_for_bucket(const document::Bucket&) const override; + class Status; class MetricUpdateHook : public framework::MetricUpdateHook { diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h index 26a0ee9098c..8fbb99dfe89 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h @@ -48,6 +48,9 @@ public: void setClusterState(std::shared_ptr<const lib::ClusterState> clusterState); const lib::ClusterState &getClusterState() const noexcept { return *_clusterState; } + const std::shared_ptr<const lib::ClusterState>& cluster_state_sp() const noexcept { + return _clusterState; + } void setDistribution(std::shared_ptr<const lib::Distribution> distribution); @@ -55,6 +58,9 @@ public: const lib::Distribution& getDistribution() const noexcept { return *_distribution; } + const std::shared_ptr<const lib::Distribution>& distribution_sp() const noexcept { + return _distribution; + } }; diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h index d9f037bb8f1..aba58e112dc 100644 --- a/storage/src/vespa/storage/distributor/distributorinterface.h +++ b/storage/src/vespa/storage/distributor/distributorinterface.h @@ -4,6 +4,7 @@ #include "bucketgctimecalculator.h" #include "distributormessagesender.h" #include "bucketownership.h" +#include "operation_routing_snapshot.h" #include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/document/bucket/bucket.h> @@ -49,6 +50,8 @@ public: */ virtual const lib::ClusterStateBundle& getClusterStateBundle() const = 0; + virtual OperationRoutingSnapshot read_snapshot_for_bucket(const document::Bucket&) const = 0; + /** * Returns true if the node is currently initializing. */ diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp index 1b88f02cac6..221c516a56e 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "bucket_space_distribution_context.h" #include "externaloperationhandler.h" #include "distributor.h" #include <vespa/document/base/documentid.h> @@ -12,7 +13,6 @@ #include <vespa/storage/distributor/operations/external/statbucketlistoperation.h> #include <vespa/storage/distributor/operations/external/removelocationoperation.h> #include <vespa/storage/distributor/operations/external/visitoroperation.h> -#include <vespa/document/util/stringutil.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/stat.h> @@ -30,11 +30,16 @@ ExternalOperationHandler::ExternalOperationHandler(Distributor& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, const MaintenanceOperationGenerator& gen, - DistributorComponentRegister& compReg) + DistributorComponentRegister& compReg, + bool enable_concurrent_gets) : DistributorComponent(owner, bucketSpaceRepo, readOnlyBucketSpaceRepo, compReg, "External operation handler"), _operationGenerator(gen), - _rejectFeedBeforeTimeReached() // At epoch -{ } + _rejectFeedBeforeTimeReached(), // At epoch + _non_main_thread_ops_mutex(), + _non_main_thread_ops_owner(owner, getClock()), + _enable_concurrent_gets(enable_concurrent_gets) +{ +} ExternalOperationHandler::~ExternalOperationHandler() = default; @@ -78,24 +83,32 @@ void ExternalOperationHandler::bounce_with_result(api::StorageCommand& cmd, cons sendUp(std::shared_ptr<api::StorageMessage>(reply.release())); } -void ExternalOperationHandler::bounce_with_wrong_distribution(api::StorageCommand& cmd) { +void ExternalOperationHandler::bounce_with_wrong_distribution(api::StorageCommand& cmd, + const lib::ClusterState& cluster_state) +{ // Distributor ownership is equal across bucket spaces, so always send back default space state. // This also helps client avoid getting confused by possibly observing different actual // (derived) state strings for global/non-global document types for the same state version. // Similarly, if we've yet to activate any version at all we send back BUSY instead // of a suspiciously empty WrongDistributionReply. // TOOD consider NOT_READY instead of BUSY once we're sure this won't cause any other issues. - const auto& cluster_state = _bucketSpaceRepo.get(document::FixedBucketSpaces::default_space()).getClusterState(); if (cluster_state.getVersion() != 0) { auto cluster_state_str = cluster_state.toString(); - LOG(debug, "Got message with wrong distribution, sending back state '%s'", cluster_state_str.c_str()); + LOG(debug, "Got %s with wrong distribution, sending back state '%s'", + cmd.toString().c_str(), cluster_state_str.c_str()); bounce_with_result(cmd, api::ReturnCode(api::ReturnCode::WRONG_DISTRIBUTION, cluster_state_str)); } else { // Only valid for empty startup state - LOG(debug, "Got message with wrong distribution, but no cluster state activated yet. Sending back BUSY"); + LOG(debug, "Got %s with wrong distribution, but no cluster state activated yet. Sending back BUSY", + cmd.toString().c_str()); bounce_with_result(cmd, api::ReturnCode(api::ReturnCode::BUSY, "No cluster state activated yet")); } } +void ExternalOperationHandler::bounce_with_wrong_distribution(api::StorageCommand& cmd) { + const auto& cluster_state = _bucketSpaceRepo.get(document::FixedBucketSpaces::default_space()).getClusterState(); + bounce_with_wrong_distribution(cmd, cluster_state); +} + void ExternalOperationHandler::bounce_with_busy_during_state_transition( api::StorageCommand& cmd, const lib::ClusterState& current_state, @@ -283,10 +296,23 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Get) { document::Bucket bucket(cmd->getBucket().getBucketSpace(), getBucketId(cmd->getDocumentId())); auto& metrics = getMetrics().gets[cmd->getLoadType()]; - bounce_or_invoke_read_only_op(*cmd, bucket, metrics, [&](auto& bucket_space_repo) { - _op = std::make_shared<GetOperation>(*this, bucket_space_repo.get(cmd->getBucket().getBucketSpace()), - cmd, metrics); - }); + auto snapshot = getDistributor().read_snapshot_for_bucket(bucket); + if (!snapshot.is_routable()) { + const auto& ctx = snapshot.context(); + if (ctx.has_pending_state_transition()) { + bounce_with_busy_during_state_transition(*cmd, *ctx.default_active_cluster_state(), + *ctx.pending_cluster_state()); + } else { + bounce_with_wrong_distribution(*cmd, *snapshot.context().default_active_cluster_state()); + metrics.failures.wrongdistributor.inc(); // TODO thread safety for updates + } + return true; + } + // The snapshot is aware of whether stale reads are enabled, so we don't have to check that here. + const auto* space_repo = snapshot.bucket_space_repo(); + assert(space_repo != nullptr); + _op = std::make_shared<GetOperation>(*this, space_repo->get(bucket.getBucketSpace()), + snapshot.steal_read_guard(), cmd, metrics); return true; } diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 655feb5d00c..9db078af198 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -8,6 +8,7 @@ #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <chrono> +#include <mutex> namespace storage { @@ -39,7 +40,8 @@ public: DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, const MaintenanceOperationGenerator&, - DistributorComponentRegister& compReg); + DistributorComponentRegister& compReg, + bool enable_concurrent_gets); ~ExternalOperationHandler() override; @@ -55,6 +57,9 @@ private: OperationSequencer _mutationSequencer; Operation::SP _op; TimePoint _rejectFeedBeforeTimeReached; + mutable std::mutex _non_main_thread_ops_mutex; + OperationOwner _non_main_thread_ops_owner; + bool _enable_concurrent_gets; template <typename Func> void bounce_or_invoke_read_only_op(api::StorageCommand& cmd, @@ -62,6 +67,8 @@ private: PersistenceOperationMetricSet& metrics, Func f); + void bounce_with_wrong_distribution(api::StorageCommand& cmd, const lib::ClusterState& cluster_state); + // Bounce with the current _default_ space cluster state. void bounce_with_wrong_distribution(api::StorageCommand& cmd); void bounce_with_busy_during_state_transition(api::StorageCommand& cmd, const lib::ClusterState& current_state, diff --git a/storage/src/vespa/storage/distributor/operation_routing_snapshot.cpp b/storage/src/vespa/storage/distributor/operation_routing_snapshot.cpp new file mode 100644 index 00000000000..ec97e51b66d --- /dev/null +++ b/storage/src/vespa/storage/distributor/operation_routing_snapshot.cpp @@ -0,0 +1,30 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "operation_routing_snapshot.h" + +namespace storage::distributor { + +OperationRoutingSnapshot::OperationRoutingSnapshot(std::shared_ptr<const BucketSpaceDistributionContext> context, + std::shared_ptr<BucketDatabase::ReadGuard> read_guard, + const DistributorBucketSpaceRepo* bucket_space_repo) + : _context(std::move(context)), + _read_guard(std::move(read_guard)), + _bucket_space_repo(bucket_space_repo) +{} + +OperationRoutingSnapshot::~OperationRoutingSnapshot() = default; + +OperationRoutingSnapshot OperationRoutingSnapshot::make_not_routable_in_state( + std::shared_ptr<const BucketSpaceDistributionContext> context) +{ + return OperationRoutingSnapshot(std::move(context), std::shared_ptr<BucketDatabase::ReadGuard>(), nullptr); +} + +OperationRoutingSnapshot OperationRoutingSnapshot::make_routable_with_guard( + std::shared_ptr<const BucketSpaceDistributionContext> context, + std::shared_ptr<BucketDatabase::ReadGuard> read_guard, + const DistributorBucketSpaceRepo& bucket_space_repo) +{ + return OperationRoutingSnapshot(std::move(context), std::move(read_guard), &bucket_space_repo); +} + +} diff --git a/storage/src/vespa/storage/distributor/operation_routing_snapshot.h b/storage/src/vespa/storage/distributor/operation_routing_snapshot.h new file mode 100644 index 00000000000..16ec8fef1c7 --- /dev/null +++ b/storage/src/vespa/storage/distributor/operation_routing_snapshot.h @@ -0,0 +1,60 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storage/bucketdb/bucketdatabase.h> +#include <memory> + +namespace storage::distributor { + +class BucketSpaceDistributionContext; +class DistributorBucketSpaceRepo; + +/** + * An "operation routing snapshot" is intended to provide a stable means of computing + * bucket routing targets and performing database lookups for a particular bucket space + * in a potentially multi-threaded setting. When using multiple threads, both the current + * cluster/distribution state as well as the underlying bucket database may change + * independent of each other when observed from any other thread than the main distributor + * thread. Additionally, the bucket management system may operate with separate read-only + * databases during state transitions, complicating things further. + * + * By using an OperationRoutingSnapshot, a caller gets a consistent view of the world + * that stays valid throughout the operation's life time. + * + * Note that holding the DB read guard should be done for as short a time as possible to + * avoid elevated memory usage caused by data stores not being able to free on-hold items. + */ +class OperationRoutingSnapshot { + std::shared_ptr<const BucketSpaceDistributionContext> _context; + std::shared_ptr<BucketDatabase::ReadGuard> _read_guard; + const DistributorBucketSpaceRepo* _bucket_space_repo; +public: + OperationRoutingSnapshot(std::shared_ptr<const BucketSpaceDistributionContext> context, + std::shared_ptr<BucketDatabase::ReadGuard> read_guard, + const DistributorBucketSpaceRepo* bucket_space_repo); + + static OperationRoutingSnapshot make_not_routable_in_state(std::shared_ptr<const BucketSpaceDistributionContext> context); + static OperationRoutingSnapshot make_routable_with_guard(std::shared_ptr<const BucketSpaceDistributionContext> context, + std::shared_ptr<BucketDatabase::ReadGuard> read_guard, + const DistributorBucketSpaceRepo& bucket_space_repo); + + OperationRoutingSnapshot(const OperationRoutingSnapshot&) noexcept = default; + OperationRoutingSnapshot& operator=(const OperationRoutingSnapshot&) noexcept = default; + OperationRoutingSnapshot(OperationRoutingSnapshot&&) noexcept = default; + OperationRoutingSnapshot& operator=(OperationRoutingSnapshot&&) noexcept = default; + + ~OperationRoutingSnapshot(); + + const BucketSpaceDistributionContext& context() const noexcept { return *_context; } + std::shared_ptr<BucketDatabase::ReadGuard> steal_read_guard() noexcept { + return std::move(_read_guard); + } + bool is_routable() const noexcept { + return (_read_guard.get() != nullptr); + } + const DistributorBucketSpaceRepo* bucket_space_repo() const noexcept { + return _bucket_space_repo; + } +}; + +} diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp index 6cfc688db0e..7ff2e298791 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp @@ -45,7 +45,8 @@ GetOperation::GroupId::operator==(const GroupId& other) const } GetOperation::GetOperation(DistributorComponent& manager, - DistributorBucketSpace &bucketSpace, + const DistributorBucketSpace &bucketSpace, + std::shared_ptr<BucketDatabase::ReadGuard> read_guard, std::shared_ptr<api::GetCommand> msg, PersistenceOperationMetricSet& metric) : Operation(), @@ -58,7 +59,7 @@ GetOperation::GetOperation(DistributorComponent& manager, _metric(metric), _operationTimer(manager.getClock()) { - assignTargetNodeGroups(); + assignTargetNodeGroups(*read_guard); } void @@ -213,13 +214,13 @@ GetOperation::sendReply(DistributorMessageSender& sender) } void -GetOperation::assignTargetNodeGroups() +GetOperation::assignTargetNodeGroups(const BucketDatabase::ReadGuard& read_guard) { document::BucketIdFactory bucketIdFactory; document::BucketId bid = bucketIdFactory.getBucketId(_msg->getDocumentId()); std::vector<BucketDatabase::Entry> entries; - _bucketSpace.getBucketDatabase().acquire_read_guard()->find_parents_and_self(bid, entries); + read_guard.find_parents_and_self(bid, entries); for (uint32_t j = 0; j < entries.size(); ++j) { const BucketDatabase::Entry& e = entries[j]; diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.h b/storage/src/vespa/storage/distributor/operations/external/getoperation.h index 3936f13077e..fe4dab5e9f2 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.h @@ -3,7 +3,7 @@ #include <vespa/storageapi/defs.h> #include <vespa/storage/distributor/operations/operation.h> -#include <vespa/storage/bucketdb/bucketcopy.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageframework/generic/clock/timer.h> @@ -23,8 +23,11 @@ class DistributorBucketSpace; class GetOperation : public Operation { public: - GetOperation(DistributorComponent& manager, DistributorBucketSpace &bucketSpace, - std::shared_ptr<api::GetCommand> msg, PersistenceOperationMetricSet& metric); + GetOperation(DistributorComponent& manager, + const DistributorBucketSpace &bucketSpace, + std::shared_ptr<BucketDatabase::ReadGuard> read_guard, + std::shared_ptr<api::GetCommand> msg, + PersistenceOperationMetricSet& metric); void onClose(DistributorMessageSender& sender) override; void onStart(DistributorMessageSender& sender) override; @@ -74,7 +77,7 @@ private: std::map<GroupId, GroupVector> _responses; DistributorComponent& _manager; - DistributorBucketSpace &_bucketSpace; + const DistributorBucketSpace &_bucketSpace; std::shared_ptr<api::GetCommand> _msg; @@ -89,7 +92,7 @@ private: void sendReply(DistributorMessageSender& sender); bool sendForChecksum(DistributorMessageSender& sender, const document::BucketId& id, GroupVector& res); - void assignTargetNodeGroups(); + void assignTargetNodeGroups(const BucketDatabase::ReadGuard& read_guard); bool copyIsOnLocalNode(const BucketCopy&) const; /** * Returns the vector index of the target to send to, or -1 if none diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index b7ebafc114c..b3326a43be2 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -178,7 +178,8 @@ TwoPhaseUpdateOperation::startSafePathUpdate(DistributorMessageSender& sender) document::Bucket bucket(_updateCmd->getBucket().getBucketSpace(), document::BucketId(0)); auto get = std::make_shared<api::GetCommand>(bucket, _updateCmd->getDocumentId(),"[all]"); copyMessageSettings(*_updateCmd, *get); - auto getOperation = std::make_shared<GetOperation>(_manager, _bucketSpace, get, _getMetric); + auto getOperation = std::make_shared<GetOperation>( + _manager, _bucketSpace, _bucketSpace.getBucketDatabase().acquire_read_guard(), get, _getMetric); GetOperation & op = *getOperation; IntermediateMessageSender intermediate(_sentMessageMap, std::move(getOperation), sender); op.start(intermediate, _manager.getClock().getTimeInMillis()); diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java index 2b5bbb188dc..9de06e7f4da 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java @@ -5,12 +5,17 @@ import ai.vespa.hosted.api.Properties; public class ApiAuthenticator implements ai.vespa.hosted.api.ApiAuthenticator { - /** Returns an authenticating controller client, using private key signatures for authentication. */ + /** Returns a controller client using mTLS if a key and certificate pair is provided, or signed requests otherwise. */ @Override public ControllerHttpClient controller() { - return ControllerHttpClient.withSignatureKey(Properties.endpoint(), - Properties.privateKeyFile(), - Properties.application()); + return Properties.certificateFile() + .map(certificateFile -> ControllerHttpClient.withKeyAndCertificate(Properties.endpoint(), + Properties.privateKeyFile(), + certificateFile)) + .orElseGet(() -> + ControllerHttpClient.withSignatureKey(Properties.endpoint(), + Properties.privateKeyFile(), + Properties.application())); } } diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java index c1cca56f1b9..c9640763ac8 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java @@ -15,6 +15,7 @@ import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Instant; import java.util.Optional; +import java.util.logging.Logger; import static ai.vespa.hosted.api.Properties.getNonBlankProperty; @@ -25,6 +26,8 @@ import static ai.vespa.hosted.api.Properties.getNonBlankProperty; */ public class EndpointAuthenticator implements ai.vespa.hosted.api.EndpointAuthenticator { + private static final Logger logger = Logger.getLogger(EndpointAuthenticator.class.getName()); + /** Don't touch. */ public EndpointAuthenticator(@SuppressWarnings("unused") SystemName __) { } @@ -35,22 +38,39 @@ public class EndpointAuthenticator implements ai.vespa.hosted.api.EndpointAuthen @Override public SSLContext sslContext() { try { + Path certificateFile = null; + Path privateKeyFile = null; Optional<String> credentialsRootProperty = getNonBlankProperty("vespa.test.credentials.root"); - if (credentialsRootProperty.isEmpty()) - return SSLContext.getDefault(); - - Path credentialsRoot = Path.of(credentialsRootProperty.get()); - Path certificateFile = credentialsRoot.resolve("cert"); - Path privateKeyFile = credentialsRoot.resolve("key"); - - X509Certificate certificate = X509CertificateUtils.fromPem(new String(Files.readAllBytes(certificateFile))); - if ( Instant.now().isBefore(certificate.getNotBefore().toInstant()) - || Instant.now().isAfter(certificate.getNotAfter().toInstant())) - throw new IllegalStateException("Certificate at '" + certificateFile + "' is valid between " + - certificate.getNotBefore() + " and " + certificate.getNotAfter() + " — not now."); + if (credentialsRootProperty.isPresent()) { + Path credentialsRoot = Path.of(credentialsRootProperty.get()); + certificateFile = credentialsRoot.resolve("cert"); + privateKeyFile = credentialsRoot.resolve("key"); + } + else { + Optional<String> certificateFileProperty = getNonBlankProperty("dataPlaneCertificateFile"); + if (certificateFileProperty.isPresent()) + certificateFile = Path.of(certificateFileProperty.get()); + Optional<String> privateKeyFileProperty = getNonBlankProperty("dataPlaneKeyFile"); + if (privateKeyFileProperty.isPresent()) + privateKeyFile = Path.of(privateKeyFileProperty.get()); + } + if (certificateFile != null && privateKeyFile != null) { + X509Certificate certificate = X509CertificateUtils.fromPem(new String(Files.readAllBytes(certificateFile))); + if ( Instant.now().isBefore(certificate.getNotBefore().toInstant()) + || Instant.now().isAfter(certificate.getNotAfter().toInstant())) + throw new IllegalStateException("Certificate at '" + certificateFile + "' is valid between " + + certificate.getNotBefore() + " and " + certificate.getNotAfter() + " — not now."); - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(new String(Files.readAllBytes(privateKeyFile))); - return new SslContextBuilder().withKeyStore(privateKey, certificate).build(); + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(new String(Files.readAllBytes(privateKeyFile))); + return new SslContextBuilder().withKeyStore(privateKey, certificate).build(); + } + logger.warning( "##################################################################################\n" + + "# Data plane key and/or certificate missing; please specify #\n" + + "# '-DdataPlaneCertificateFile=/path/to/certificate' and #\n" + + "# '-DdataPlaneKeyFile=/path/to/private_key. #\n" + + "# Trying the default SSLContext, but this will most likely cause HTTP error 401. #\n" + + "##################################################################################"); + return SSLContext.getDefault(); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java index bda7e41c19b..4cc92828b0e 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java @@ -36,9 +36,10 @@ public abstract class ClientBase implements AutoCloseable { protected ClientBase(String userAgent, Supplier<SSLContext> sslContextSupplier, - ClientExceptionFactory exceptionFactory) { + ClientExceptionFactory exceptionFactory, + HostnameVerifier hostnameVerifier) { this.exceptionFactory = exceptionFactory; - this.client = createHttpClient(userAgent, sslContextSupplier); + this.client = createHttpClient(userAgent, sslContextSupplier, hostnameVerifier); } protected <T> T execute(HttpUriRequest request, ResponseHandler<T> responseHandler) { @@ -74,11 +75,11 @@ public abstract class ClientBase implements AutoCloseable { return statusCode>=200 && statusCode<300; } - private static CloseableHttpClient createHttpClient(String userAgent, Supplier<SSLContext> sslContextSupplier) { + private static CloseableHttpClient createHttpClient(String userAgent, Supplier<SSLContext> sslContextSupplier, HostnameVerifier hostnameVerifier) { return HttpClientBuilder.create() .setRetryHandler(new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true)) .setUserAgent(userAgent) - .setSSLSocketFactory(new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(sslContextSupplier), (HostnameVerifier)null)) + .setSSLSocketFactory(new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(sslContextSupplier), hostnameVerifier)) .setDefaultRequestConfig(RequestConfig.custom() .setConnectTimeout((int) Duration.ofSeconds(10).toMillis()) .setConnectionRequestTimeout((int)Duration.ofSeconds(10).toMillis()) diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index da3bd18440b..7b5427216a1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.common.ClientBase; import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; @@ -45,7 +44,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } private DefaultZmsClient(URI zmsUrl, AthenzIdentity identity, Supplier<SSLContext> sslContextSupplier) { - super("vespa-zms-client", sslContextSupplier, ZmsClientException::new); + super("vespa-zms-client", sslContextSupplier, ZmsClientException::new, null); this.zmsUrl = addTrailingSlash(zmsUrl); this.identity = identity; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index 8bd0d0b50d4..45597cbad08 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -26,6 +26,7 @@ import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; +import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; import java.io.IOException; import java.net.URI; @@ -49,15 +50,19 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { private final URI ztsUrl; public DefaultZtsClient(URI ztsUrl, SSLContext sslContext) { - this(ztsUrl, () -> sslContext); + this(ztsUrl, () -> sslContext, null); } public DefaultZtsClient(URI ztsUrl, ServiceIdentityProvider identityProvider) { - this(ztsUrl, identityProvider::getIdentitySslContext); + this(ztsUrl, identityProvider::getIdentitySslContext, null); } - private DefaultZtsClient(URI ztsUrl, Supplier<SSLContext> sslContextSupplier) { - super("vespa-zts-client", sslContextSupplier, ZtsClientException::new); + public DefaultZtsClient(URI ztsUrl, ServiceIdentityProvider identityProvider, HostnameVerifier hostnameVerifier) { + this(ztsUrl, identityProvider::getIdentitySslContext, hostnameVerifier); + } + + private DefaultZtsClient(URI ztsUrl, Supplier<SSLContext> sslContextSupplier, HostnameVerifier hostnameVerifier) { + super("vespa-zts-client", sslContextSupplier, ZtsClientException::new, hostnameVerifier); this.ztsUrl = addTrailingSlash(ztsUrl); } |