diff options
author | Martin Polden <martin.polden@gmail.com> | 2016-11-16 13:16:51 +0100 |
---|---|---|
committer | Martin Polden <martin.polden@gmail.com> | 2016-11-16 14:11:55 +0100 |
commit | 6caeb3e00f9848859c72fba9747cf613281e1f43 (patch) | |
tree | 1a53b01e03376ad0475c60bd2097356c41fb143d /node-repository | |
parent | e747a1c5e2bbbc03ad6c294cf1cd02c825e3e23b (diff) |
Persist IP address in node-repo
Diffstat (limited to 'node-repository')
18 files changed, 232 insertions, 51 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 3a52e51b6c0..956d0a31bd2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -24,6 +24,7 @@ import java.util.Optional; public final class Node { private final String id; + private final String ipAddress; private final String hostname; private final String openStackId; private final Optional<String> parentHostname; @@ -39,21 +40,22 @@ public final class Node { private Optional<Allocation> allocation; /** Creates a node in the initial state (provisioned) */ - public static Node create(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { - return new Node(openStackId, hostname, parentHostname, flavor, Status.initial(), State.provisioned, + public static Node create(String openStackId, String ipAddress, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { + return new Node(openStackId, ipAddress, hostname, parentHostname, flavor, Status.initial(), State.provisioned, Optional.empty(), History.empty(), type); } /** Do not use. Construct nodes by calling {@link NodeRepository#createNode} */ - public Node(String openStackId, String hostname, Optional<String> parentHostname, + private Node(String openStackId, String ipAddress, String hostname, Optional<String> parentHostname, Flavor flavor, Status status, State state, Allocation allocation, History history, NodeType type) { - this(openStackId, hostname, parentHostname, flavor, status, state, Optional.of(allocation), history, type); + this(openStackId, ipAddress, hostname, parentHostname, flavor, status, state, Optional.of(allocation), history, type); } - public Node(String openStackId, String hostname, Optional<String> parentHostname, + public Node(String openStackId, String ipAddress, String hostname, Optional<String> parentHostname, Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type) { Objects.requireNonNull(openStackId, "A node must have an openstack id"); + requireNonEmptyString(ipAddress, "A node must have an IP address"); requireNonEmptyString(hostname, "A node must have a hostname"); requireNonEmptyString(parentHostname, "A parent host name must be a proper value"); Objects.requireNonNull(flavor, "A node must have a flavor"); @@ -64,6 +66,7 @@ public final class Node { Objects.requireNonNull(type, "A null node type is not permitted"); this.id = hostname; + this.ipAddress = ipAddress; this.hostname = hostname; this.parentHostname = parentHostname; this.openStackId = openStackId; @@ -81,6 +84,9 @@ public final class Node { */ public String id() { return id; } + /** Returns the IP address of this node */ + public String ipAddress() { return ipAddress; } + /** Returns the host name of this node */ public String hostname() { return hostname; } @@ -142,22 +148,22 @@ public final class Node { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { - return new Node(openStackId, hostname, parentHostname, flavor, status, state, allocation, history, type); + return new Node(openStackId, ipAddress, hostname, parentHostname, flavor, status, state, allocation, history, type); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { - return new Node(openStackId, hostname, parentHostname, flavor, status, state, allocation, history, type); + return new Node(openStackId, ipAddress, hostname, parentHostname, flavor, status, state, allocation, history, type); } /** Returns a node with the flavor assigned to the given value */ public Node with(Flavor flavor) { - return new Node(openStackId, hostname, parentHostname, flavor, status, state, allocation, history, type); + return new Node(openStackId, ipAddress, hostname, parentHostname, flavor, status, state, allocation, history, type); } /** Returns a copy of this with the current reboot generation set to generation */ public Node withReboot(Generation generation) { - return new Node(openStackId, hostname, parentHostname, flavor, status.withReboot(generation), state, + return new Node(openStackId, ipAddress, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, history, type); } @@ -182,18 +188,18 @@ public final class Node { * Do not use this to allocate a node. */ public Node with(Allocation allocation) { - return new Node(openStackId, hostname, parentHostname, flavor, status, state, allocation, history, type); + return new Node(openStackId, ipAddress, hostname, parentHostname, flavor, status, state, allocation, history, type); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { - return new Node(openStackId, hostname, Optional.of(parentHostname), flavor, status, state, + return new Node(openStackId, ipAddress, hostname, Optional.of(parentHostname), flavor, status, state, allocation, history, type); } /** Returns a copy of this node with the given history. */ public Node with(History history) { - return new Node(openStackId, hostname, parentHostname, flavor, status, state, allocation, history, type); + return new Node(openStackId, ipAddress, hostname, parentHostname, flavor, status, state, allocation, history, type); } private void requireNonEmptyString(Optional<String> value, String message) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 7a6e9f3223d..875605ff93a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; +import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import java.time.Clock; import java.util.ArrayList; @@ -57,6 +58,7 @@ import java.util.stream.Collectors; public class NodeRepository extends AbstractComponent { private final CuratorDatabaseClient zkClient; + private final NameResolver nameResolver; /** * Creates a node repository form a zookeeper provider. @@ -64,15 +66,16 @@ public class NodeRepository extends AbstractComponent { */ @Inject public NodeRepository(NodeFlavors flavors, Curator curator, Zone zone) { - this(flavors, curator, Clock.systemUTC(), zone); + this(flavors, curator, Clock.systemUTC(), zone, new NameResolver() {} /* use default implementation */); } /** * Creates a node repository form a zookeeper provider and a clock instance * which will be used for time-sensitive decisions. */ - public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone) { - this.zkClient = new CuratorDatabaseClient(flavors, curator, clock, zone); + public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver) { + this.zkClient = new CuratorDatabaseClient(flavors, curator, clock, zone, nameResolver); + this.nameResolver = nameResolver; // read and write all nodes to make sure they are stored in the latest version of the serialized format for (Node.State state : Node.State.values()) @@ -118,9 +121,9 @@ public class NodeRepository extends AbstractComponent { // ----------------- Node lifecycle ----------------------------------------------------------- /** Creates a new node object, without adding it to the node repo */ - public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, + public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { - return Node.create(openStackId, hostname, parentHostname, flavor, type); + return Node.create(openStackId, nameResolver.getByNameOrThrow(hostname), hostname, parentHostname, flavor, type); } /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index f30686f4d79..bdba2dc6de5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -10,12 +10,11 @@ import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; - -import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.hosted.provision.node.Status; import java.time.Clock; @@ -51,8 +50,8 @@ public class CuratorDatabaseClient { private final Zone zone; - public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone) { - this.nodeSerializer = new NodeSerializer(flavors); + public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver) { + this.nodeSerializer = new NodeSerializer(flavors, nameResolver); this.zone = zone; jsonMapper.registerModule(new JodaModule()); this.curatorDatabase = new CuratorDatabase(curator, root, /* useCache: */ false); @@ -146,7 +145,8 @@ public class CuratorDatabaseClient { CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); for (Node node : nodes) { - Node newNode = new Node(node.openStackId(), node.hostname(), node.parentHostname(), node.flavor(), + Node newNode = new Node(node.openStackId(), node.ipAddress(), node.hostname(), + node.parentHostname(), node.flavor(), newNodeStatus(node, toState), toState, toState.isAllocated() ? node.allocation() : Optional.empty(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java new file mode 100644 index 00000000000..e37f8ccc3f9 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Interface for a basic name to IP address resolver. Default implementation delegates to + * {@link java.net.InetAddress#getByName(String)}. + * + * @author mpolden + */ +public interface NameResolver { + + /** Resolve IP address from given host name */ + default String getByNameOrThrow(String hostname) { + try { + return InetAddress.getByName(hostname).getHostAddress(); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index f2d50e76baa..1eb4be2fbcd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -40,8 +40,11 @@ public class NodeSerializer { /** The configured node flavors */ private final NodeFlavors flavors; + private final NameResolver nameResolver; + // Node fields private static final String hostnameKey = "hostname"; + private static final String ipAddressKey = "ipAddress"; private static final String openStackIdKey = "openStackId"; private static final String parentHostnameKey = "parentHostname"; private static final String historyKey = "history"; @@ -76,8 +79,9 @@ public class NodeSerializer { // ---------------- Serialization ---------------------------------------------------- - public NodeSerializer(NodeFlavors flavors) { + public NodeSerializer(NodeFlavors flavors, NameResolver nameResolver) { this.flavors = flavors; + this.nameResolver = nameResolver; } public byte[] toJson(Node node) { @@ -93,6 +97,7 @@ public class NodeSerializer { private void toSlime(Node node, Cursor object) { object.setString(hostnameKey, node.hostname()); + object.setString(ipAddressKey, node.ipAddress()); object.setString(openStackIdKey, node.openStackId()); node.parentHostname().ifPresent(hostname -> object.setString(parentHostnameKey, hostname)); object.setString(flavorKey, node.flavor().name()); @@ -141,6 +146,7 @@ public class NodeSerializer { private Node nodeFromSlime(Node.State state, Inspector object) { return new Node(object.field(openStackIdKey).asString(), + ipAddressFromResolverOrSlime(object), object.field(hostnameKey).asString(), parentHostnameFromSlime(object), flavorFromSlime(object), @@ -217,6 +223,14 @@ public class NodeSerializer { else return Optional.empty(); } + + // TODO: Remove this and use the field directly after 6.48 has been deployed everywhere + private String ipAddressFromResolverOrSlime(Inspector object) { + if (!object.field(ipAddressKey).valid()) { + return nameResolver.getByNameOrThrow(object.field("hostname").asString()); + } + return object.field(ipAddressKey).asString(); + } private Optional<Status.HardwareFailureType> hardwareFailureFromSlime(Inspector object) { if ( ! object.valid()) return Optional.empty(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java new file mode 100644 index 00000000000..761fb73757f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.testutils; + +import com.yahoo.vespa.hosted.provision.persistence.NameResolver; + +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * A mock DNS resolver. Can be configured to only answer specific lookups or any lookup. + * + * @author mpolden + */ +public class MockNameResolver implements NameResolver { + + private boolean allowInvocation = true; + private boolean mockAnyLookup = false; + private final Map<String, String> records = new HashMap<>(); + + public MockNameResolver addRecord(String hostname, String ipAddress) { + records.put(hostname, ipAddress); + return this; + } + + public MockNameResolver reset() { + this.allowInvocation = true; + this.mockAnyLookup = false; + this.records.clear(); + return this; + } + + public MockNameResolver failIfInvoked() { + this.allowInvocation = false; + return this; + } + + public MockNameResolver mockAnyLookup() { + this.mockAnyLookup = true; + return this; + } + + @Override + public String getByNameOrThrow(String hostname) { + if (!allowInvocation) { + throw new IllegalStateException("Expected getByName to not be invoked for hostname: " + hostname); + } + if (mockAnyLookup) { + return UUID.randomUUID().toString(); + } + if (records.containsKey(hostname)) { + return records.get(hostname); + } + throw new RuntimeException(new UnknownHostException("Could not resolve: " + hostname)); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 707d9207d8d..e6901c6e159 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -39,7 +39,8 @@ public class MockNodeRepository extends NodeRepository { * @param flavors flavors to have in node repo */ public MockNodeRepository(NodeFlavors flavors) throws Exception { - super(flavors, new MockCurator(), Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone()); + super(flavors, new MockCurator(), Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), + new MockNameResolver().mockAnyLookup()); this.flavors = flavors; populate(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 475cd047960..41d8219f623 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.node.Flavor; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import java.time.Clock; import java.util.Collections; @@ -30,7 +31,8 @@ public class NodeRepositoryTester { clock = new ManualClock(); curator = new MockCurator(); curator.setConnectionSpec("server1:1234,server2:5678"); - nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone()); + nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), + new MockNameResolver().mockAnyLookup()); } public NodeRepository nodeRepository() { return nodeRepository; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java index 8da137d4c86..46bee56d80b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Duration; @@ -46,7 +47,8 @@ public class ApplicationMaintainerTest { ManualClock clock = new ManualClock(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, + new MockNameResolver().mockAnyLookup()); createReadyNodes(15, nodeRepository, nodeFlavors); createHostNodes(2, nodeRepository, nodeFlavors); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index c691bb076c9..6fc3cbaa596 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Duration; @@ -81,7 +82,8 @@ public class FailedExpirerTest { private NodeRepository failureScenarioIn(SystemName system, Environment environment) { ManualClock clock = new ManualClock(); NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone()); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), + new MockNameResolver().mockAnyLookup()); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock); List<Node> nodes = new ArrayList<>(3); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 0ebcdd1b7bc..cf68cedc564 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -33,6 +33,7 @@ import com.yahoo.vespa.hosted.provision.node.Flavor; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; @@ -86,7 +87,7 @@ public class NodeFailTester { public NodeFailTester() { clock = new ManualClock(); curator = new MockCurator(); - nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone); + nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup()); provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index e40b39e6182..ae444cbcc28 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -13,11 +13,11 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; -import java.time.Duration; - import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -37,7 +37,8 @@ public class ReservationExpirerTest { public void ensure_reservation_times_out() throws InterruptedException { ManualClock clock = new ManualClock(); NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); - NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone()); + NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(), + new MockNameResolver().mockAnyLookup()); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), clock); List<Node> nodes = new ArrayList<>(2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 9570bdda8f3..ad99e114340 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -16,12 +16,13 @@ import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Duration; @@ -45,7 +46,8 @@ public class RetiredExpirerTest { ManualClock clock = new ManualClock(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, + new MockNameResolver().mockAnyLookup()); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); createReadyNodes(7, nodeRepository, nodeFlavors); @@ -83,7 +85,8 @@ public class RetiredExpirerTest { ManualClock clock = new ManualClock(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, + new MockNameResolver().mockAnyLookup()); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); createReadyNodes(8, nodeRepository, nodeFlavors); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java index 2c1c1a6ca72..42654bce8f5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java @@ -10,15 +10,18 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; -import static org.junit.Assert.assertEquals; +import java.time.Clock; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; +import static org.junit.Assert.assertEquals; + /** * @author oyving */ @@ -28,7 +31,8 @@ public class ProvisionMetricsTest { public void test_registered_metric() throws InterruptedException { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); Curator curator = new MockCurator(); - NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Zone.defaultZone()); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), + new MockNameResolver().mockAnyLookup()); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(Collections.singletonList(node)); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index 6ce40047352..e5d67e5d789 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -11,23 +11,26 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Clock; import java.util.List; + import static org.junit.Assert.assertEquals; /** - * @author Oyvind Gronnesby + * @author mpolden */ public class CuratorDatabaseClientTest { - private Curator curator = new MockCurator(); - private CuratorDatabaseClient zkClient = new CuratorDatabaseClient(FlavorConfigBuilder.createDummies("default"), - curator, Clock.systemUTC(), Zone.defaultZone()); + private final Curator curator = new MockCurator(); + private final CuratorDatabaseClient zkClient = new CuratorDatabaseClient( + FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), + new MockNameResolver().mockAnyLookup()); @Test - public void ensure_can_read_stored_host_information() throws Exception { + public void can_read_stored_host_information() throws Exception { String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"flavor\":\"default\",\"created\":1421054425159, \"type\":\"host\"}"; curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/ready/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes()); @@ -36,9 +39,8 @@ public class CuratorDatabaseClientTest { assertEquals(NodeType.host, allocatedNodes.get(0).type()); } - /** Test that locks can be acquired and released */ @Test - public void testLocking() { + public void locks_can_be_acquired_and_released() { ApplicationId app = ApplicationId.from(TenantName.from("testTenant"), ApplicationName.from("testApp"), InstanceName.from("testInstance")); try (CuratorMutex mutex1 = zkClient.lock(app)) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 56d19446cd9..d5368179474 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -18,8 +18,11 @@ import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; +import org.junit.Before; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Optional; @@ -29,13 +32,20 @@ import static org.junit.Assert.assertTrue; /** * @author bratseth + * @author mpolden */ public class SerializationTest { private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "large", "ugccloud-container"); - private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors); + private final MockNameResolver nameResolver = new MockNameResolver(); + private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors, nameResolver); private final ManualClock clock = new ManualClock(); + @Before + public void before() { + nameResolver.reset(); + } + @Test public void testProvisionedNodeSerialization() { Node node = createNode(); @@ -101,6 +111,7 @@ public class SerializationTest { @Test public void testRebootAndRestartAndTypeNoCurrentValuesSerialization() { + nameResolver.addRecord("myHostname", "127.0.0.1"); String nodeData = "{\n" + " \"type\" : \"tenant\",\n" + @@ -140,6 +151,7 @@ public class SerializationTest { // TODO: Remove when 6.31 is deployed everywhere @Test public void testLegacyFlavor() { + nameResolver.addRecord("myHostname", "127.0.0.1"); String nodeData = "{\n" + " \"type\" : \"tenant\",\n" + @@ -196,7 +208,8 @@ public class SerializationTest { @Test public void testAssimilatedDeserialization() { - Node node = nodeSerializer.fromJson(Node.State.active, "{\"type\":\"tenant\",\"hostname\":\"assimilate2.vespahosted.corp.bf1.yahoo.com\",\"openStackId\":\"\",\"flavor\":\"ugccloud-container\",\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"ugc-assimilate\",\"instanceId\":\"default\",\"serviceId\":\"container/ugccloud-container/0/0\",\"restartGeneration\":0}}\n".getBytes()); + nameResolver.addRecord("assimilate2.vespahosted.yahoo.tld", "127.0.0.1"); + Node node = nodeSerializer.fromJson(Node.State.active, "{\"type\":\"tenant\",\"hostname\":\"assimilate2.vespahosted.yahoo.tld\",\"openStackId\":\"\",\"flavor\":\"ugccloud-container\",\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"ugc-assimilate\",\"instanceId\":\"default\",\"serviceId\":\"container/ugccloud-container/0/0\",\"restartGeneration\":0}}\n".getBytes()); assertEquals(0, node.history().events().size()); assertTrue(node.allocation().isPresent()); assertEquals("ugccloud-container", node.allocation().get().membership().cluster().id().value()); @@ -243,14 +256,54 @@ public class SerializationTest { @Test public void serialize_parentHostname() { final String parentHostname = "parent.yahoo.com"; - Node node = Node.create("myId", "myHostname", Optional.of(parentHostname), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); + Node node = Node.create("myId", "myIp", "myHostname", Optional.of(parentHostname), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); assertEquals(parentHostname, deserializedNode.parentHostname().get()); } + + @Test + public void resolves_hostname_when_deserializing() throws Exception { + nameResolver.addRecord("node1.yahoo.tld", "127.0.0.1"); + byte[] nodeWithoutIp = createNodeJson("node1.yahoo.tld"); + Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeWithoutIp); + assertEquals("127.0.0.1", deserializedNode.ipAddress()); + } + + @Test + public void throws_when_hostname_cannot_be_resolved() throws Exception { + byte[] nodeWithoutIp = createNodeJson("node2.yahoo.tld"); + try { + nodeSerializer.fromJson(State.provisioned, nodeWithoutIp); + assertTrue("Expected exception to be thrown", false); + } catch (RuntimeException ignored) { + } + } + + @Test + public void resolver_is_not_invoked_when_ip_address_is_present() throws Exception { + nameResolver.failIfInvoked(); + byte[] nodeWithIp = createNodeJson("node3.yahoo.tld", "127.0.0.3"); + Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeWithIp); + assertEquals("127.0.0.3", deserializedNode.ipAddress()); + } + + private byte[] createNodeJson(String hostname, String ipAddress) { + return ("{\"hostname\":\"" + hostname + "\"," + + (ipAddress.isEmpty() ? "" : "\"ipAddress\":\"" + ipAddress + "\",") + + "\"openStackId\":\"myId\"," + + "\"flavor\":\"default\",\"rebootGeneration\":0," + + "\"currentRebootGeneration\":0,\"failCount\":0,\"history\":[],\"type\":\"tenant\"}") + .getBytes(StandardCharsets.UTF_8); + } + + private byte[] createNodeJson(String hostname) { + return createNodeJson(hostname, ""); + } + private Node createNode() { - return Node.create("myId", "myHostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host); + return Node.create("myId", "myIp", "myHostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 02389c229b3..4352a64a3f9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.node.Flavor; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import java.io.IOException; import java.time.temporal.TemporalAmount; @@ -61,7 +62,8 @@ public class ProvisioningTester implements AutoCloseable { try { nodeFlavors = new NodeFlavors(createConfig()); clock = new ManualClock(); - nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone); + nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, + new MockNameResolver().mockAnyLookup()); provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock); capacityPolicies = new CapacityPolicies(zone, nodeFlavors); provisionLogger = new NullProvisionLogger(); @@ -75,7 +77,8 @@ public class ProvisioningTester implements AutoCloseable { try { nodeFlavors = new NodeFlavors(config); clock = new ManualClock(); - nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone); + nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, + new MockNameResolver().mockAnyLookup()); provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock); capacityPolicies = new CapacityPolicies(zone, nodeFlavors); provisionLogger = new NullProvisionLogger(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java index 5374cd52e0f..90672cfbb47 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Clock; @@ -72,12 +73,14 @@ public class RestApiTest { } // Instantiated by DI from application package above + @SuppressWarnings("unused") public static class MockNodeRepository extends NodeRepository { private static final NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); public MockNodeRepository() throws Exception { - super(flavors, new MockCurator(), Clock.systemUTC(), Zone.defaultZone()); + super(flavors, new MockCurator(), Clock.systemUTC(), Zone.defaultZone(), + new MockNameResolver().mockAnyLookup()); populate(); } |