diff options
11 files changed, 132 insertions, 79 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java index 11fd352bcc9..974e5203e76 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java @@ -27,9 +27,16 @@ public class CloudAccount extends PatternedStringWrapper<CloudAccount> { return this.equals(empty); } + /** Returns true if this is an enclave account. */ + public boolean isEnclave(Zone zone) { + return !isUnspecified() && + zone.system().isPublic() && + !equals(zone.cloud().account()); + } + public static CloudAccount from(String cloudAccount) { return switch (cloudAccount) { - // TODO: Remove "default" as e.g. it is a valid GCP project ID + // Tenants are allowed to specify "default" in services.xml. case "", "default" -> empty; default -> new CloudAccount(cloudAccount, AWS_ACCOUNT_ID + "|" + GCP_PROJECT_ID, "cloud account"); }; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index c9f53b69dc6..e3e192eba5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -284,7 +284,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { NodePrioritizer prioritizer = new NodePrioritizer(nodesAndHosts, applicationId, clusterSpec, nodeSpec, wantedGroups, true, nodeRepository().nameResolver(), nodeRepository().nodes(), nodeRepository().resourcesCalculator(), - nodeRepository().spareCount()); + nodeRepository().spareCount(), nodeSpec.cloudAccount().isEnclave(nodeRepository().zone())); List<NodeCandidate> nodeCandidates = prioritizer.collect(List.of()); MutableInteger index = new MutableInteger(0); return nodeCandidates diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index 3fb07496714..c606ede05d1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -82,8 +82,9 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { /** Verify DNS configuration of given nodes */ private void verifyDns(List<Node> nodes) { for (var node : nodes) { + boolean enclave = node.cloudAccount().isEnclave(nodeRepository().zone()); for (var ipAddress : node.ipConfig().primary()) { - IP.verifyDns(node.hostname(), ipAddress, nodeRepository().nameResolver()); + IP.verifyDns(node.hostname(), ipAddress, nodeRepository().nameResolver(), !enclave); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java index 532e6747d9a..d7ef2228960 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java @@ -8,43 +8,10 @@ import java.util.Objects; * * @author hakon */ -public class Address { - - private final String hostname; - - public Address(String hostname) { - this.hostname = validateHostname(hostname, "hostname"); - } - - public String hostname() { - return hostname; - } - - @Override - public String toString() { - return "Address{" + - "hostname='" + hostname + '\'' + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Address address = (Address) o; - return hostname.equals(address.hostname); - } - - @Override - public int hashCode() { - return Objects.hash(hostname); - } - - private String validateHostname(String value, String name) { - Objects.requireNonNull(value, name + " cannot be null"); - if (value.isEmpty()) { - throw new IllegalArgumentException(name + " cannot be empty"); - } - return value; +public record Address(String hostname) { + public Address { + Objects.requireNonNull(hostname, "hostname cannot be null"); + if (hostname.isEmpty()) + throw new IllegalArgumentException("hostname cannot be empty"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 2693fff3b39..c0d0b220767 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -203,7 +203,7 @@ public class IP { return new IpAddresses(addresses, Protocol.ipv4); } - // If we're dual-stacked, we must must have an equal number of addresses of each protocol. + // If we're dual-stacked, we must have an equal number of addresses of each protocol. if (ipv4AddrCount == ipv6AddrCount) { return new IpAddresses(addresses, Protocol.dualStack); } @@ -215,9 +215,15 @@ public class IP { } public enum Protocol { - dualStack, - ipv4, - ipv6 + dualStack("dual-stack"), + ipv4("IPv4-only"), + ipv6("IPv6-only"); + + private final String description; + + Protocol(String description) { this.description = description; } + + public String getDescription() { return description; } } @Override @@ -266,7 +272,7 @@ public class IP { * @param nodes a locked list of all nodes in the repository * @return an allocation from the pool, if any can be made */ - public Optional<Allocation> findAllocation(LockedNodeList nodes, NameResolver resolver) { + public Optional<Allocation> findAllocation(LockedNodeList nodes, NameResolver resolver, boolean hasPtr) { if (ipAddresses.asSet().isEmpty()) { // IP addresses have not yet been resolved and should be done later. return findUnusedAddressStream(nodes) @@ -274,10 +280,15 @@ public class IP { .findFirst(); } + if (!hasPtr) { + // Without PTR records (reverse IP mapping): Ensure only forward resolving from hostnames. + return findUnusedAddressStream(nodes).findFirst().map(address -> Allocation.fromAddress(address, resolver, ipAddresses.protocol)); + } + if (ipAddresses.protocol == IpAddresses.Protocol.ipv4) { return findUnusedIpAddresses(nodes).stream() - .findFirst() - .map(addr -> Allocation.ofIpv4(addr, resolver)); + .findFirst() + .map(addr -> Allocation.ofIpv4(addr, resolver)); } var unusedAddresses = findUnusedIpAddresses(nodes); @@ -364,8 +375,8 @@ public class IP { /** * Allocate an IPv6 address. * - * A successful allocation is guaranteed to have an IPv6 address, but may also have an IPv4 address if the - * hostname of the IPv6 address has an A record. + * <p>A successful allocation is guaranteed to have an IPv6 address, but may also have an IPv4 address if the + * hostname of the IPv6 address has an A record.</p> * * @param ipv6Address Unassigned IPv6 address * @param resolver DNS name resolver to use @@ -415,6 +426,33 @@ public class IP { return new Allocation(hostname4, Optional.of(addresses.get(0)), Optional.empty()); } + private static Allocation fromAddress(Address address, NameResolver resolver, IpAddresses.Protocol protocol) { + // Resolve both A and AAAA to verify they match the protocol and to avoid surprises later on. + + Optional<String> ipv4Address = resolveOptional(address.hostname(), resolver, RecordType.A); + if (protocol != IpAddresses.Protocol.ipv6 && ipv4Address.isEmpty()) + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " did not resolve to an IPv4 address"); + if (protocol == IpAddresses.Protocol.ipv6 && ipv4Address.isPresent()) + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " has an IPv4 address: " + ipv4Address.get()); + + Optional<String> ipv6Address = resolveOptional(address.hostname(), resolver, RecordType.AAAA); + if (protocol != IpAddresses.Protocol.ipv4 && ipv6Address.isEmpty()) + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " did not resolve to an IPv6 address"); + if (protocol == IpAddresses.Protocol.ipv4 && ipv6Address.isPresent()) + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " has an IPv6 address: " + ipv6Address.get()); + + return new Allocation(address.hostname(), ipv4Address, ipv6Address); + } + + private static Optional<String> resolveOptional(String hostname, NameResolver resolver, RecordType recordType) { + Set<String> values = resolver.resolve(hostname, recordType); + return switch (values.size()) { + case 0 -> Optional.empty(); + case 1 -> Optional.of(values.iterator().next()); + default -> throw new IllegalArgumentException("Hostname " + hostname + " resolved to more than one " + recordType.description() + ": " + values); + }; + } + private static Allocation ofAddress(Address address) { return new Allocation(address.hostname(), Optional.empty(), Optional.empty()); } @@ -460,20 +498,22 @@ public class IP { } /** Verify DNS configuration of given hostname and IP address */ - public static void verifyDns(String hostname, String ipAddress, NameResolver resolver) { + public static void verifyDns(String hostname, String ipAddress, NameResolver resolver, boolean hasPtr) { RecordType recordType = isV6(ipAddress) ? RecordType.AAAA : RecordType.A; Set<String> addresses = resolver.resolve(hostname, recordType); if (!addresses.equals(Set.of(ipAddress))) throw new IllegalArgumentException("Expected " + hostname + " to resolve to " + ipAddress + ", but got " + addresses); - Optional<String> reverseHostname = resolver.resolveHostname(ipAddress); - if (reverseHostname.isEmpty()) - throw new IllegalArgumentException(ipAddress + " did not resolve to a hostname"); + if (hasPtr) { + Optional<String> reverseHostname = resolver.resolveHostname(ipAddress); + if (reverseHostname.isEmpty()) + throw new IllegalArgumentException(ipAddress + " did not resolve to a hostname"); - if (!reverseHostname.get().equals(hostname)) - throw new IllegalArgumentException(ipAddress + " resolved to " + reverseHostname.get() + - ", which does not match expected hostname " + hostname); + if (!reverseHostname.get().equals(hostname)) + throw new IllegalArgumentException(ipAddress + " resolved to " + reverseHostname.get() + + ", which does not match expected hostname " + hostname); + } } /** Convert IP address to string. This uses :: for zero compression in IPv6 addresses. */ @@ -483,7 +523,7 @@ public class IP { /** Returns whether given string is an IPv4 address */ public static boolean isV4(String ipAddress) { - return ipAddress.contains("."); + return !isV6(ipAddress) && ipAddress.contains("."); } /** Returns whether given string is an IPv6 address */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java index 59f29d7629a..c32c96bc0dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java @@ -23,17 +23,18 @@ public interface NameResolver { /** DNS record types */ enum RecordType { - A("A"), - AAAA("AAAA"); + A("A", "IPv4 address"), + AAAA("AAAA", "IPv6 address"); private final String value; + private final String description; - public String value() { - return value; - } + public String value() { return value; } + public String description() { return description; } - RecordType(String value) { + RecordType(String value, String description) { this.value = value; + this.description = description; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 1a3ac17c7ef..15e508fadf1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -165,7 +165,8 @@ public class GroupPreparer { nodeRepository.nameResolver(), nodeRepository.nodes(), nodeRepository.resourcesCalculator(), - nodeRepository.spareCount()); + nodeRepository.spareCount(), + requestedNodes.cloudAccount().isEnclave(nodeRepository.zone())); allocation.offer(prioritizer.collect(surplusActiveNodes)); return allocation; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 4ad52a51df9..fa07782057b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -267,8 +267,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat Node parent, boolean violatesSpares, LockedNodeList allNodes, - NameResolver nameResolver) { - return new VirtualNodeCandidate(resources, freeParentCapacity, parent, violatesSpares, true, allNodes, nameResolver); + NameResolver nameResolver, + boolean hasPtrRecord) { + return new VirtualNodeCandidate(resources, freeParentCapacity, parent, violatesSpares, true, allNodes, nameResolver, hasPtrRecord); } public static NodeCandidate createNewExclusiveChild(Node node, Node parent) { @@ -366,6 +367,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Needed to construct the node */ private final LockedNodeList allNodes; private final NameResolver nameResolver; + private final boolean hasPtrRecord; private VirtualNodeCandidate(NodeResources resources, NodeResources freeParentCapacity, @@ -373,11 +375,13 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat boolean violatesSpares, boolean exclusiveSwitch, LockedNodeList allNodes, - NameResolver nameResolver) { + NameResolver nameResolver, + boolean hasPtrRecord) { super(freeParentCapacity, Optional.of(parent), violatesSpares, exclusiveSwitch, false, true, false); this.resources = resources; this.allNodes = allNodes; this.nameResolver = nameResolver; + this.hasPtrRecord = hasPtrRecord; } @Override @@ -416,7 +420,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat public NodeCandidate withNode() { Optional<IP.Allocation> allocation; try { - allocation = parent.get().ipConfig().pool().findAllocation(allNodes, nameResolver); + allocation = parent.get().ipConfig().pool().findAllocation(allNodes, nameResolver, hasPtrRecord); if (allocation.isEmpty()) return new InvalidNodeCandidate(resources, freeParentCapacity, parent.get(), "No addresses available on parent host"); } catch (Exception e) { @@ -440,7 +444,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) { - return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, allNodes, nameResolver); + return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, allNodes, nameResolver, hasPtrRecord); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index c1d65e0df4e..79d05ce5c97 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -45,10 +45,11 @@ public class NodePrioritizer { private final boolean topologyChange; private final int currentClusterSize; private final Set<Node> spareHosts; + private final boolean enclave; public NodePrioritizer(NodesAndHosts<LockedNodeList> allNodesAndHosts, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, int wantedGroups, boolean dynamicProvisioning, NameResolver nameResolver, Nodes nodes, - HostResourcesCalculator hostResourcesCalculator, int spareCount) { + HostResourcesCalculator hostResourcesCalculator, int spareCount, boolean enclave) { this.allNodesAndHosts = allNodesAndHosts; this.capacity = new HostCapacity(this.allNodesAndHosts, hostResourcesCalculator); this.requestedNodes = nodeSpec; @@ -60,6 +61,7 @@ public class NodePrioritizer { capacity.findSpareHosts(this.allNodesAndHosts.nodes().asList(), spareCount); this.nameResolver = nameResolver; this.nodes = nodes; + this.enclave = enclave; NodeList nodesInCluster = this.allNodesAndHosts.nodes().owner(application).type(clusterSpec.type()).cluster(clusterSpec.id()); NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired(); @@ -152,7 +154,8 @@ public class NodePrioritizer { host, spareHosts.contains(host), allNodesAndHosts.nodes(), - nameResolver)); + nameResolver, + !enclave)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java index dbc74f32f6b..94cb05d20cc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import java.net.UnknownHostException; import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -12,6 +15,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; /** * A mock DNS resolver. Can be configured to only answer specific lookups or any lookup. @@ -73,7 +77,16 @@ public class MockNameResolver implements NameResolver { @Override public Set<String> resolve(String name, RecordType first, RecordType... rest) { - return resolveAll(name); + var types = EnumSet.of(first, rest); + + return resolveAll(name) + .stream() + .filter(addressString -> { + if (types.contains(RecordType.A) && IP.isV4(addressString)) return true; + if (types.contains(RecordType.AAAA) && IP.isV6(addressString)) return true; + return false; + }) + .collect(Collectors.toSet()); } @Override diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index f4610e722a8..c26ffdaa023 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -85,7 +85,7 @@ public class IPTest { resolver.addReverseRecord("::1", "host3"); resolver.addReverseRecord("::2", "host1"); - Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver); + Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver, true); assertEquals(Optional.of("::1"), allocation.get().ipv6Address()); assertFalse(allocation.get().ipv4Address().isPresent()); assertEquals("host3", allocation.get().hostname()); @@ -93,7 +93,7 @@ public class IPTest { // Allocation fails if DNS record is missing resolver.removeRecord("host3"); try { - pool.findAllocation(emptyList, resolver); + pool.findAllocation(emptyList, resolver, true); fail("Expected exception"); } catch (Exception e) { assertEquals("java.net.UnknownHostException: Could not resolve: host3", e.getMessage()); @@ -103,7 +103,7 @@ public class IPTest { @Test public void test_find_allocation_ipv4_only() { var pool = testPool(false); - var allocation = pool.findAllocation(emptyList, resolver); + var allocation = pool.findAllocation(emptyList, resolver, true); assertFalse("Found allocation", allocation.isEmpty()); assertEquals(Optional.of("127.0.0.1"), allocation.get().ipv4Address()); assertTrue("No IPv6 address", allocation.get().ipv6Address().isEmpty()); @@ -112,7 +112,7 @@ public class IPTest { @Test public void test_find_allocation_dual_stack() { IP.Pool pool = testPool(true); - Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver); + Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver, true); assertEquals(Optional.of("::1"), allocation.get().ipv6Address()); assertEquals("127.0.0.2", allocation.get().ipv4Address().get()); assertEquals("host3", allocation.get().hostname()); @@ -123,7 +123,7 @@ public class IPTest { IP.Pool pool = testPool(true); resolver.addRecord("host3", "127.0.0.127"); try { - pool.findAllocation(emptyList, resolver); + pool.findAllocation(emptyList, resolver, true); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("Hostname host3 resolved to more than 1 IPv4 address: [127.0.0.2, 127.0.0.127]", @@ -137,7 +137,7 @@ public class IPTest { resolver.removeRecord("127.0.0.2") .addReverseRecord("127.0.0.2", "host5"); try { - pool.findAllocation(emptyList, resolver); + pool.findAllocation(emptyList, resolver, true); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("Hostnames resolved from each IP address do not point to the same hostname " + @@ -145,6 +145,22 @@ public class IPTest { } } + @Test + public void test_enclave() { + // In Enclave, the hosts and their nodes have only public IPv6 addresses, + // and DNS has AAAA records without PTR records. + + resolver.addRecord("host1", "2600:1f10:::1") + .addRecord("node1", "2600:1f10:::2") + .addRecord("node2", "2600:1f10:::3"); + + IP.Config config = IP.Config.of(Set.of("2600:1f10:::1"), + Set.of("2600:1f10:::2", "2600:1f10:::3"), + List.of(new Address("node1"), new Address("node2"))); + IP.Pool pool = config.pool(); + Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver, false); + } + private IP.Pool testPool(boolean dualStack) { var addresses = new LinkedHashSet<String>(); addresses.add("127.0.0.1"); |