diff options
20 files changed, 306 insertions, 99 deletions
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index edda5593e79..0a757b5b17b 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -57,7 +57,7 @@ public class IdentityDocumentGeneratorTest { Generation.initial(), false); Node parentNode = Node.create("ostkid", - new IP.Config(Set.of("127.0.0.1"), Set.of()), + IP.Config.ofEmptyPool(Set.of("127.0.0.1")), parentHostname, new MockNodeFlavors().getFlavorOrThrow("default"), NodeType.host).build(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 298404256b9..d0ee6229428 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -55,7 +55,7 @@ public final class Node implements Nodelike { /** Creates a node builder in the initial state (reserved) */ public static Node.Builder createDockerNode(Set<String> ipAddresses, String hostname, String parentHostname, NodeResources resources, NodeType type) { return new Node.Builder("fake-" + hostname, hostname, new Flavor(resources), State.reserved, type) - .ipConfig(ipAddresses, Set.of()) + .ipConfig(IP.Config.ofEmptyPool(ipAddresses)) .parentHostname(parentHostname); } @@ -573,8 +573,8 @@ public final class Node implements Nodelike { return this; } - public Builder ipConfig(Set<String> primary, Set<String> pool) { - this.ipConfig = new IP.Config(primary, pool); + public Builder ipConfigWithEmptyPool(Set<String> primary) { + this.ipConfig = IP.Config.ofEmptyPool(primary); return this; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index e5202dfd3c0..6094b497fff 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -135,12 +135,12 @@ public class CapacityChecker { for (var host : hosts) { NodeResources hostResources = host.flavor().resources(); int occupiedIps = 0; - Set<String> ipPool = host.ipConfig().pool().asSet(); + Set<String> ipPool = host.ipConfig().pool().getIpSet(); for (var child : nodeChildren.get(host)) { hostResources = hostResources.subtract(child.resources().justNumbers()); occupiedIps += child.ipConfig().primary().stream().filter(ipPool::contains).count(); } - availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().asSet().size() - occupiedIps)); + availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().getIpSet().size() - occupiedIps)); } return availableResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java new file mode 100644 index 00000000000..bdef06f1070 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import java.util.Objects; + +/** + * Address info about a container that might run on a host. + * + * @author hakon + */ +public class Address { + private final String hostname; + + public Address(String hostname) { + this.hostname = validateHostname(hostname, "hostname"); + } + + public String hostname() { + return hostname; + } + + @Override + public String toString() { + return "Address{" + + "hostname='" + hostname + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Address address = (Address) o; + return hostname.equals(address.hostname); + } + + @Override + public int hashCode() { + return Objects.hash(hostname); + } + + private String validateHostname(String value, String name) { + Objects.requireNonNull(value, name + " cannot be null"); + if (value.isEmpty()) { + throw new IllegalArgumentException(name + " cannot be empty"); + } + return value; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 955936931be..41d6c1e5425 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -60,15 +60,29 @@ public class IP { /** IP configuration of a node */ public static class Config { - public static final Config EMPTY = new Config(Set.of(), Set.of()); + public static final Config EMPTY = Config.ofEmptyPool(Set.of()); private final Set<String> primary; private final Pool pool; - /** DO NOT USE in non-test code. Public for serialization purposes. */ + public static Config ofEmptyPool(Set<String> primary) { + return new Config(primary, Set.of(), List.of()); + } + + public static Config of(Set<String> primary, Set<String> ipPool, List<Address> addressPool) { + return new Config(primary, ipPool, addressPool); + } + + /** LEGACY TEST CONSTRUCTOR - use of() variants and/or the with- methods. */ public Config(Set<String> primary, Set<String> pool) { + this(primary, pool, List.of()); + } + + /** DO NOT USE: Public for NodeSerializer. */ + public Config(Set<String> primary, Set<String> pool, List<Address> addresses) { this.primary = ImmutableSet.copyOf(Objects.requireNonNull(primary, "primary must be non-null")); - this.pool = Pool.of(Objects.requireNonNull(pool, "pool must be non-null")); + this.pool = Pool.of(Objects.requireNonNull(pool, "pool must be non-null"), + Objects.requireNonNull(addresses, "addresses must be non-null")); } /** The primary addresses of this. These addresses are used when communicating with the node itself */ @@ -82,13 +96,13 @@ public class IP { } /** Returns a copy of this with pool set to given value */ - public Config with(Pool pool) { - return new Config(primary, pool.asSet()); + public Config withPool(Pool pool) { + return new Config(primary, pool.getIpSet(), pool.getAddressList()); } /** Returns a copy of this with pool set to given value */ - public Config with(Set<String> primary) { - return new Config(primary, pool.asSet()); + public Config withPrimary(Set<String> primary) { + return new Config(primary, pool.getIpSet(), pool.getAddressList()); } @Override @@ -107,7 +121,7 @@ public class IP { @Override public String toString() { - return String.format("ip config primary=%s pool=%s", primary, pool.asSet()); + return String.format("ip config primary=%s pool=%s", primary, pool.getIpSet()); } /** @@ -124,8 +138,8 @@ public class IP { var addresses = new HashSet<>(node.ipConfig().primary()); var otherAddresses = new HashSet<>(other.ipConfig().primary()); if (node.type().isHost()) { // Addresses of a host can never overlap with any other nodes - addresses.addAll(node.ipConfig().pool().asSet()); - otherAddresses.addAll(other.ipConfig().pool().asSet()); + addresses.addAll(node.ipConfig().pool().getIpSet()); + otherAddresses.addAll(other.ipConfig().pool().getIpSet()); } otherAddresses.retainAll(addresses); if (!otherAddresses.isEmpty()) @@ -157,18 +171,18 @@ public class IP { } /** A list of IP addresses and their protocol */ - public static class Addresses { + public static class IpAddresses { - private final Set<String> addresses; + private final Set<String> ipAddresses; private final Protocol protocol; - private Addresses(Set<String> addresses, Protocol protocol) { - this.addresses = ImmutableSet.copyOf(Objects.requireNonNull(addresses, "addresses must be non-null")); + private IpAddresses(Set<String> ipAddresses, Protocol protocol) { + this.ipAddresses = ImmutableSet.copyOf(Objects.requireNonNull(ipAddresses, "addresses must be non-null")); this.protocol = Objects.requireNonNull(protocol, "type must be non-null"); } public Set<String> asSet() { - return addresses; + return ipAddresses; } /** The protocol of addresses in this */ @@ -177,20 +191,20 @@ public class IP { } /** Create addresses of the given set */ - private static Addresses of(Set<String> addresses) { + private static IpAddresses of(Set<String> addresses) { long ipv6AddrCount = addresses.stream().filter(IP::isV6).count(); if (ipv6AddrCount == addresses.size()) { // IPv6-only - return new Addresses(addresses, Protocol.ipv6); + return new IpAddresses(addresses, Protocol.ipv6); } long ipv4AddrCount = addresses.stream().filter(IP::isV4).count(); if (ipv4AddrCount == addresses.size()) { // IPv4-only - return new Addresses(addresses, Protocol.ipv4); + return new IpAddresses(addresses, Protocol.ipv4); } // If we're dual-stacked, we must must have an equal number of addresses of each protocol. if (ipv4AddrCount == ipv6AddrCount) { - return new Addresses(addresses, Protocol.dualStack); + return new IpAddresses(addresses, Protocol.dualStack); } throw new IllegalArgumentException(String.format("Dual-stacked IP address list must have an " + @@ -208,15 +222,28 @@ public class IP { } /** - * A pool of IP addresses from which an allocation can be made. + * A pool of addresses from which an allocation can be made. * * Addresses in this are available for use by Docker containers */ public static class Pool { - private final Addresses addresses; + private final IpAddresses ipAddresses; + private final List<Address> addresses; - private Pool(Addresses addresses) { + /** Creates an empty pool. */ + public static Pool of() { + return of(Set.of(), List.of()); + } + + /** Create a new pool containing given ipAddresses */ + public static Pool of(Set<String> ipAddresses, List<Address> addresses) { + IpAddresses ips = IpAddresses.of(ipAddresses); + return new Pool(ips, addresses); + } + + private Pool(IpAddresses ipAddresses, List<Address> addresses) { + this.ipAddresses = Objects.requireNonNull(ipAddresses, "ipAddresses must be non-null"); this.addresses = Objects.requireNonNull(addresses, "addresses must be non-null"); } @@ -227,6 +254,12 @@ public class IP { * @return an allocation from the pool, if any can be made */ public Optional<Allocation> findAllocation(LockedNodeList nodes, NameResolver resolver) { + if (ipAddresses.protocol == IpAddresses.Protocol.ipv4) { + return findUnused(nodes).stream() + .findFirst() + .map(addr -> Allocation.ofIpv4(addr, resolver)); + } + var unusedAddresses = findUnused(nodes); var allocation = unusedAddresses.stream() .filter(IP::isV6) @@ -248,64 +281,47 @@ public class IP { * @param nodes a list of all nodes in the repository */ public Set<String> findUnused(NodeList nodes) { - var unusedAddresses = new LinkedHashSet<>(asSet()); - nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> asSet().contains(ip))) + var unusedAddresses = new LinkedHashSet<>(getIpSet()); + nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> getIpSet().contains(ip))) .forEach(node -> unusedAddresses.removeAll(node.ipConfig().primary())); return Collections.unmodifiableSet(unusedAddresses); } - public Set<String> asSet() { - return addresses.asSet(); + public IpAddresses.Protocol getProtocol() { + return ipAddresses.protocol; } - public boolean isEmpty() { - return asSet().isEmpty(); + public Set<String> getIpSet() { + return ipAddresses.asSet(); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Pool that = (Pool) o; - return Objects.equals(addresses, that.addresses); + public List<Address> getAddressList() { + return addresses; } - @Override - public int hashCode() { - return Objects.hash(addresses); + public boolean isEmpty() { + return getIpSet().isEmpty(); } - /** Create a new pool containing given ipAddresses */ - public static Pool of(Set<String> ipAddresses) { - var addresses = Addresses.of(ipAddresses); - if (addresses.protocol() == Addresses.Protocol.ipv4) { - return new Ipv4Pool(addresses); - } - return new Pool(addresses); + public Pool withIpAddresses(Set<String> ipAddresses) { + return Pool.of(ipAddresses, addresses); } - /** Validates and returns the given IP address pool */ - public static Set<String> require(Set<String> pool) { - return of(pool).asSet(); + public Pool withAddresses(List<Address> addresses) { + return Pool.of(ipAddresses.ipAddresses, addresses); } - } - - /** A pool of IPv4-only addresses from which an allocation can be made. */ - public static class Ipv4Pool extends Pool { - - private Ipv4Pool(Addresses addresses) { - super(addresses); - if (addresses.protocol() != Addresses.Protocol.ipv4) { - throw new IllegalArgumentException("Protocol of addresses must be " + Addresses.Protocol.ipv4); - } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Pool that = (Pool) o; + return Objects.equals(ipAddresses, that.ipAddresses); } @Override - public Optional<Allocation> findAllocation(LockedNodeList nodes, NameResolver resolver) { - return findUnused(nodes).stream() - .findFirst() - .map(addr -> Allocation.ofIpv4(addr, resolver)); + public int hashCode() { + return Objects.hash(ipAddresses); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 0f171351fba..cd794531473 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -26,6 +26,7 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -66,6 +67,8 @@ public class NodeSerializer { private static final String hostnameKey = "hostname"; private static final String ipAddressesKey = "ipAddresses"; private static final String ipAddressPoolKey = "additionalIpAddresses"; + private static final String containersKey = "containers"; + private static final String containerHostnameKey = "hostname"; private static final String idKey = "openStackId"; private static final String parentHostnameKey = "parentHostname"; private static final String historyKey = "history"; @@ -146,7 +149,8 @@ public class NodeSerializer { private void toSlime(Node node, Cursor object) { object.setString(hostnameKey, node.hostname()); toSlime(node.ipConfig().primary(), object.setArray(ipAddressesKey)); - toSlime(node.ipConfig().pool().asSet(), object.setArray(ipAddressPoolKey)); + toSlime(node.ipConfig().pool().getIpSet(), object.setArray(ipAddressPoolKey)); + toSlime(node.ipConfig().pool().getAddressList(), object); object.setString(idKey, node.id()); node.parentHostname().ifPresent(hostname -> object.setString(parentHostnameKey, hostname)); toSlime(node.flavor(), object); @@ -214,6 +218,14 @@ public class NodeSerializer { ipAddresses.stream().map(IP::parse).sorted(IP.NATURAL_ORDER).map(IP::asString).forEach(array::addString); } + private void toSlime(List<Address> addresses, Cursor object) { + if (addresses.isEmpty()) return; + Cursor addressCursor = object.setArray(containersKey); + addresses.forEach(address -> { + addressCursor.addObject().setString(containerHostnameKey, address.hostname()); + }); + } + // ---------------- Deserialization -------------------------------------------------- public Node fromJson(Node.State state, byte[] data) { @@ -232,7 +244,8 @@ public class NodeSerializer { Flavor flavor = flavorFromSlime(object); return new Node(object.field(idKey).asString(), new IP.Config(ipAddressesFromSlime(object, ipAddressesKey), - ipAddressesFromSlime(object, ipAddressPoolKey)), + ipAddressesFromSlime(object, ipAddressPoolKey), + addressesFromSlime(object)), object.field(hostnameKey).asString(), parentHostnameFromSlime(object), flavor, @@ -358,6 +371,16 @@ public class NodeSerializer { return ipAddresses.build(); } + private List<Address> addressesFromSlime(Inspector object) { + Inspector addressesField = object.field(containersKey); + if (addressesField.children() == 0) + return List.of(); + List<Address> addresses = new ArrayList<>(addressesField.children()); + addressesField.traverse((ArrayTraverser) (i, elem) -> + addresses.add(new Address(elem.field(containerHostnameKey).asString()))); + return addresses; + } + private Optional<String> modelNameFromSlime(Inspector object) { if (object.field(modelNameKey).valid()) { return Optional.of(object.field(modelNameKey).asString()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index f2755176124..832310cf2c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -15,6 +15,7 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.IP; @@ -26,6 +27,7 @@ import java.io.InputStream; import java.io.UncheckedIOException; import java.time.Clock; import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -116,9 +118,11 @@ public class NodePatcher { case "parentHostname" : return node.withParentHostname(asString(value)); case "ipAddresses" : - return IP.Config.verify(node.with(node.ipConfig().with(asStringSet(value))), memoizedNodes.get()); + return IP.Config.verify(node.with(node.ipConfig().withPrimary(asStringSet(value))), memoizedNodes.get()); case "additionalIpAddresses" : - return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), memoizedNodes.get()); + return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withIpAddresses(asStringSet(value)))), memoizedNodes.get()); + case "additionalHostnames" : + return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withAddresses(asAddressList(value)))), memoizedNodes.get()); case WANT_TO_RETIRE : case WANT_TO_DEPROVISION : boolean wantToRetire = asOptionalBoolean(root.field(WANT_TO_RETIRE)).orElse(node.status().wantToRetire()); @@ -212,6 +216,22 @@ public class NodePatcher { return strings; } + private List<Address> asAddressList(Inspector field) { + if ( ! field.type().equals(Type.ARRAY)) + throw new IllegalArgumentException("Expected an ARRAY value, got a " + field.type()); + + List<Address> addresses = new ArrayList<>(field.entries()); + for (int i = 0; i < field.entries(); i++) { + Inspector entry = field.entry(i); + if ( ! entry.type().equals(Type.STRING)) + throw new IllegalArgumentException("Expected a STRING value, got a " + entry.type()); + Address address = new Address(entry.asString()); + addresses.add(address); + } + + return addresses; + } + private Node patchRequiredDiskSpeed(Node node, String value) { Optional<Allocation> allocation = node.allocation(); if (allocation.isPresent()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 39375494d01..3a9246379cf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -15,6 +15,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -186,7 +187,8 @@ class NodesResponse extends HttpResponse { object.setBool("wantToDeprovision", node.status().wantToDeprovision()); toSlime(node.history(), object.setArray("history")); ipAddressesToSlime(node.ipConfig().primary(), object.setArray("ipAddresses")); - ipAddressesToSlime(node.ipConfig().pool().asSet(), object.setArray("additionalIpAddresses")); + ipAddressesToSlime(node.ipConfig().pool().getIpSet(), object.setArray("additionalIpAddresses")); + addressesToSlime(node.ipConfig().pool().getAddressList(), object); node.reports().toSlime(object, "reports"); node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); @@ -229,6 +231,13 @@ class NodesResponse extends HttpResponse { ipAddresses.forEach(array::addString); } + private void addressesToSlime(List<Address> addresses, Cursor object) { + if (addresses.isEmpty()) return; + // When/if Address becomes richer: add another field (e.g. "addresses") and expand to array of objects + Cursor addressesArray = object.setArray("additionalHostnames"); + addresses.forEach(address -> addressesArray.addString(address.hostname())); + } + private String lastElement(String path) { if (path.endsWith("/")) path = path.substring(0, path.length()-1); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 59604d094fa..304cebb3c01 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -257,7 +257,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); Node.Builder builder = Node.create(inspector.field("openStackId").asString(), - new IP.Config(ipAddresses, ipAddressPool), + IP.Config.of(ipAddresses, ipAddressPool, List.of()), inspector.field("hostname").asString(), flavorFromSlime(inspector), nodeTypeFromSlime(inspector.field("type"))); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index fb78e76f0c7..b24d2417db5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -206,11 +206,11 @@ public class MockNodeRepository extends NodeRepository { private IP.Config ipConfig(int nodeIndex, int primarySize, int poolSize) { var primary = new LinkedHashSet<String>(); - var pool = new LinkedHashSet<String>(); + var ipPool = new LinkedHashSet<String>(); for (int i = 1; i <= primarySize + poolSize; i++) { var set = primary; if (i > primarySize) { - set = pool; + set = ipPool; } var rootName = "test-node-primary"; if (i > primarySize) { @@ -226,7 +226,7 @@ public class MockNodeRepository extends NodeRepository { set.add(ipv4Address); } } - return new IP.Config(primary, pool); + return IP.Config.of(primary, ipPool, List.of()); } private IP.Config ipConfig(int nodeIndex) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index c08e51bf0cc..478376bc0cd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -66,9 +66,9 @@ public class DynamicProvisioningMaintainerTest { assertTrue("No IP addresses assigned", Stream.of(host3, host4, host41).map(node -> node.ipConfig().primary()).allMatch(Set::isEmpty)); - Node host3new = host3.with(host3.ipConfig().with(Set.of("::3:0"))); - Node host4new = host4.with(host4.ipConfig().with(Set.of("::4:0"))); - Node host41new = host41.with(host41.ipConfig().with(Set.of("::4:1", "::4:2"))); + Node host3new = host3.with(host3.ipConfig().withPrimary(Set.of("::3:0"))); + Node host4new = host4.with(host4.ipConfig().withPrimary(Set.of("::4:0"))); + Node host41new = host41.with(host41.ipConfig().withPrimary(Set.of("::4:1", "::4:2"))); tester.maintainer.maintain(); assertEquals(host3new, tester.nodeRepository.getNode("host3").get()); @@ -290,7 +290,7 @@ public class DynamicProvisioningMaintainerTest { Generation.initial(), false)); Node.Builder builder = Node.create("fake-id-" + hostname, hostname, flavor, state, nodeType) - .ipConfig(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of()); + .ipConfigWithEmptyPool(state == Node.State.active ? Set.of("::1") : Set.of()); parentHostname.ifPresent(builder::parentHostname); allocation.ifPresent(builder::allocation); return builder.build(); @@ -382,16 +382,18 @@ public class DynamicProvisioningMaintainerTest { if (node.parentHostname().isPresent()) return node; int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", "")); Set<String> addresses = Set.of("::" + hostIndex + ":0"); - Set<String> pool = new HashSet<>(); + Set<String> ipAddressPool = new HashSet<>(); if (!behaviours.contains(Behaviour.failDnsUpdate)) { nameResolver.addRecord(node.hostname(), addresses.iterator().next()); for (int i = 1; i <= 2; i++) { String ip = "::" + hostIndex + ":" + i; - pool.add(ip); + ipAddressPool.add(ip); nameResolver.addRecord(node.hostname() + "-" + i, ip); } } - return node.with(node.ipConfig().with(addresses).with(IP.Pool.of(pool))); + + IP.Pool pool = node.ipConfig().pool().withIpAddresses(ipAddressPool); + return node.with(node.ipConfig().withPrimary(addresses).withPool(pool)); } enum Behaviour { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index a209ea206d3..fb9c1ad0e5a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -177,7 +177,7 @@ public class IPTest { } IP.Pool pool = node.ipConfig().pool(); - assertNotEquals(dualStack, pool instanceof IP.Ipv4Pool); + assertNotEquals(dualStack, pool.getProtocol() == IP.IpAddresses.Protocol.ipv4); return pool; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 10e3592e548..8b9d60aeaf4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -21,6 +21,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.Node.State; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.History; @@ -249,15 +250,19 @@ public class NodeSerializerTest { public void serialize_ip_address_pool() { Node node = createNode(); - // Test round-trip with IP address pool - node = node.with(node.ipConfig().with(IP.Pool.of(Set.of("::1", "::2", "::3")))); + // Test round-trip with address pool + node = node.with(node.ipConfig().withPool(IP.Pool.of( + Set.of("::1", "::2", "::3"), + List.of(new Address("a"), new Address("b"), new Address("c"))))); Node copy = nodeSerializer.fromJson(node.state(), nodeSerializer.toJson(node)); - assertEquals(node.ipConfig().pool().asSet(), copy.ipConfig().pool().asSet()); + assertEquals(node.ipConfig().pool().getIpSet(), copy.ipConfig().pool().getIpSet()); + assertEquals(Set.copyOf(node.ipConfig().pool().getAddressList()), Set.copyOf(copy.ipConfig().pool().getAddressList())); - // Test round-trip without IP address pool (handle empty pool) + // Test round-trip without address pool (handle empty pool) node = createNode(); copy = nodeSerializer.fromJson(node.state(), nodeSerializer.toJson(node)); - assertEquals(node.ipConfig().pool().asSet(), copy.ipConfig().pool().asSet()); + assertEquals(node.ipConfig().pool().getIpSet(), copy.ipConfig().pool().getIpSet()); + assertEquals(Set.copyOf(node.ipConfig().pool().getAddressList()), Set.copyOf(copy.ipConfig().pool().getAddressList())); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index b76588b7036..b59ef458d25 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -11,8 +11,9 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.node.IP; -import javax.swing.JFrame; +import javax.swing.*; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -76,7 +77,7 @@ public class AllocationSimulator { Node.Builder builder = Node.create("fake", hostname, flavor, parent.isPresent() ? Node.State.ready : Node.State.active, parent.isPresent() ? NodeType.tenant : NodeType.host) - .ipConfig(Set.of("127.0.0.1"), parent.isPresent() ? Set.of() : getAdditionalIP()); + .ipConfig(IP.Config.of(Set.of("127.0.0.1"), parent.isPresent() ? Set.of() : getAdditionalIP(), List.of())); parent.ifPresent(builder::parentHostname); allocation(tenant, flavor).ifPresent(builder::allocation); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index 4891387e9e8..c6e89680e85 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -43,9 +43,9 @@ public class HostCapacityTest { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); // Create three docker hosts - host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); - host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); - host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); + host1 = Node.create("host1", IP.Config.of(Set.of("::1"), generateIPs(2, 4), List.of()), "host1", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); + host2 = Node.create("host2", IP.Config.of(Set.of("::11"), generateIPs(12, 3), List.of()), "host2", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); + host3 = Node.create("host3", IP.Config.of(Set.of("::21"), generateIPs(22, 1), List.of()), "host3", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); // Add two containers to host1 var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", resources1, NodeType.tenant).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index fe1fb43dcb9..c51ef7250e2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -256,7 +256,7 @@ public class LoadBalancerProvisionerTest { private void assignIps(List<Node> nodes) { try (var lock = tester.nodeRepository().lockUnallocated()) { for (int i = 0; i < nodes.size(); i++) { - tester.nodeRepository().write(nodes.get(i).with(IP.Config.EMPTY.with(Set.of("127.0.0." + i))), lock); + tester.nodeRepository().write(nodes.get(i).with(IP.Config.EMPTY.withPrimary(Set.of("127.0.0." + i))), lock); } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java index 7f637ba8804..8aa362aa932 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java @@ -133,7 +133,7 @@ public class NodeCandidateTest { private static Node node(String hostname, Node.State state) { return Node.create(hostname, hostname, new Flavor(new NodeResources(2, 2, 2, 2)), state, NodeType.tenant) - .ipConfig(Set.of("::1"), Set.of()).build(); + .ipConfigWithEmptyPool(Set.of("::1")).build(); } private static NodeCandidate node(String hostname, @@ -143,9 +143,9 @@ public class NodeCandidateTest { boolean exclusiveSwitch) { Node node = Node.create(hostname, hostname, new Flavor(nodeResources), Node.State.ready, NodeType.tenant) .parentHostname(hostname + "parent") - .ipConfig(Set.of("::1"), Set.of()).build(); + .ipConfigWithEmptyPool(Set.of("::1")).build(); Node parent = Node.create(hostname + "parent", hostname, new Flavor(totalHostResources), Node.State.ready, NodeType.host) - .ipConfig(Set.of("::1"), Set.of()).build(); + .ipConfigWithEmptyPool(Set.of("::1")).build(); return new NodeCandidate.ConcreteNodeCandidate(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), false, exclusiveSwitch, false, true, false); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 82b20bcae14..2fe39780cf5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -894,7 +894,7 @@ public class ProvisioningTest { Flavor flavor = tester.nodeRepository().flavors().getFlavorOrThrow("default"); List<Node> nodes = List.of( Node.create("cfghost1", new IP.Config(Set.of("::1:0"), Set.of("::1:1")), "cfghost1", flavor, NodeType.confighost).build(), - Node.create("cfghost2", new IP.Config(Set.of("::2:0"), Set.of("::2:1")), "cfghost2", flavor, NodeType.confighost).ipConfig(Set.of("::2:0"), Set.of("::2:1")).build(), + Node.create("cfghost2", new IP.Config(Set.of("::2:0"), Set.of("::2:1")), "cfghost2", flavor, NodeType.confighost).ipConfig(IP.Config.of(Set.of("::2:0"), Set.of("::2:1"), List.of())).build(), Node.create("cfg1", new IP.Config(Set.of("::1:1"), Set.of()), "cfg1", flavor, NodeType.config).parentHostname("cfghost1").build(), Node.create("cfg2", new IP.Config(Set.of("::2:1"), Set.of()), "cfg2", flavor, NodeType.config).parentHostname("cfghost2").build()); tester.nodeRepository().setReady(tester.nodeRepository().addNodes(nodes, Agent.system), Agent.system, ProvisioningTest.class.getSimpleName()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 97f850102fc..a98d383e219 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -360,6 +360,23 @@ public class NodesV2ApiTest { } @Test + public void patch_hostnames() throws IOException { + assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4.json"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"additionalHostnames\": [\"a\",\"b\"]}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + + assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-with-hostnames.json"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"additionalHostnames\": []}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + + assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4.json"); + } + + @Test public void post_controller_node() throws Exception { String data = "[{\"hostname\":\"controller1.yahoo.com\", \"openStackId\":\"fake-controller1.yahoo.com\"," + createIpAddresses("127.0.0.1") + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json new file mode 100644 index 00000000000..40b5f29c13f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json @@ -0,0 +1,65 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", + "id": "host4.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host4.yahoo.com", + "parentHostname": "dockerhost1.yahoo.com", + "openStackId": "node4", + "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "vespaVersion": "6.41.0", + "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.41.0", + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.4.1", + "::4:1" + ], + "additionalIpAddresses": [], + "additionalHostnames": ["a","b"] +} |