From 5928ec9b766ba31f9199040e7fde0684f46a5482 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 17 Apr 2018 22:56:44 +0200 Subject: Method to add nodes to node-repository --- .../admin/configserver/noderepository/AddNode.java | 82 ++++++++++++++++++++++ .../noderepository/NodeRepository.java | 3 +- .../noderepository/RealNodeRepository.java | 25 +++++++ .../noderepository/RealNodeRepositoryTest.java | 60 ++++++++++------ .../node/admin/integrationTests/NodeRepoMock.java | 6 +- 5 files changed, 151 insertions(+), 25 deletions(-) create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java new file mode 100644 index 00000000000..3b5220192d4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java @@ -0,0 +1,82 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +import com.yahoo.config.provision.NodeType; + +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +/** + * @author freva + */ +public class AddNode { + + public final String hostname; + public final Optional parentHostname; + public final String nodeFlavor; + public final NodeType nodeType; + public final Set ipAddresses; + public final Set additionalIpAddresses; + + /** + * Constructor for a host node (has no parent) + */ + public AddNode(String hostname, String nodeFlavor, NodeType nodeType, Set ipAddresses, Set additionalIpAddresses) { + this(hostname, Optional.empty(), nodeFlavor, nodeType, ipAddresses, additionalIpAddresses); + } + + /** + * Constructor for a child node (Must set parentHostname, no additionalIpAddresses) + */ + public AddNode(String hostname, String parentHostname, String nodeFlavor, NodeType nodeType, Set ipAddresses) { + this(hostname, Optional.of(parentHostname), nodeFlavor, nodeType, ipAddresses, Collections.emptySet()); + } + + public AddNode(String hostname, Optional parentHostname, String nodeFlavor, NodeType nodeType, Set ipAddresses, Set additionalIpAddresses) { + this.hostname = hostname; + this.parentHostname = parentHostname; + this.nodeFlavor = nodeFlavor; + this.nodeType = nodeType; + this.ipAddresses = ipAddresses; + this.additionalIpAddresses = additionalIpAddresses; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AddNode addNode = (AddNode) o; + + if (!hostname.equals(addNode.hostname)) return false; + if (!parentHostname.equals(addNode.parentHostname)) return false; + if (!nodeFlavor.equals(addNode.nodeFlavor)) return false; + if (nodeType != addNode.nodeType) return false; + if (!ipAddresses.equals(addNode.ipAddresses)) return false; + return additionalIpAddresses.equals(addNode.additionalIpAddresses); + } + + @Override + public int hashCode() { + int result = hostname.hashCode(); + result = 31 * result + parentHostname.hashCode(); + result = 31 * result + nodeFlavor.hashCode(); + result = 31 * result + nodeType.hashCode(); + result = 31 * result + ipAddresses.hashCode(); + result = 31 * result + additionalIpAddresses.hashCode(); + return result; + } + + @Override + public String toString() { + return "AddNode{" + + "hostname='" + hostname + '\'' + + ", parentHostname=" + parentHostname + + ", nodeFlavor='" + nodeFlavor + '\'' + + ", nodeType=" + nodeType + + ", ipAddresses=" + ipAddresses + + ", additionalIpAddresses=" + additionalIpAddresses + + '}'; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java index 77ab1ac2482..c915e4b432f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java @@ -10,13 +10,14 @@ import com.yahoo.vespa.hosted.provision.Node; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * @author stiankri */ public interface NodeRepository { + void addNodes(List nodes); + List getNodes(String baseHostName); Optional getNode(String hostName); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index dbb31a5573d..23351bd47ad 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; +import com.google.common.base.Strings; import com.google.common.net.InetAddresses; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.DockerImage; @@ -39,6 +40,18 @@ public class RealNodeRepository implements NodeRepository { this.configServerApi = configServerApi; } + @Override + public void addNodes(List nodes) { + List nodesToPost = nodes.stream() + .map(RealNodeRepository::nodeRepositoryNodeFromAddNode) + .collect(Collectors.toList()); + + NodeMessageResponse response = configServerApi.post("/nodes/v2/node", nodesToPost, NodeMessageResponse.class); + if (!Strings.isNullOrEmpty(response.errorCode)) { + throw new RuntimeException("Failed to add nodes to node-repo: " + response.message + " " + response.errorCode); + } + } + @Override public List getNodes(String baseHostName) { return getNodes(Optional.of(baseHostName), Collections.emptyList()); @@ -206,4 +219,16 @@ public class RealNodeRepository implements NodeRepository { Optional.ofNullable(node.hardwareDivergence), Optional.ofNullable(node.parentHostname)); } + + private static NodeRepositoryNode nodeRepositoryNodeFromAddNode(AddNode addNode) { + NodeRepositoryNode node = new NodeRepositoryNode(); + node.openStackId = "fake-" + addNode.hostname; + node.hostname = addNode.hostname; + node.parentHostname = addNode.parentHostname.orElse(null); + node.flavor = addNode.nodeFlavor; + node.type = addNode.nodeType.name(); + node.ipAddresses = addNode.ipAddresses; + node.additionalIpAddresses = addNode.additionalIpAddresses; + return node; + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 269c79b125c..ca177c96979 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; +import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; import com.yahoo.vespa.hosted.provision.Node; @@ -19,13 +21,17 @@ import java.io.IOException; import java.net.ServerSocket; import java.net.URI; import java.time.Instant; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Optional; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -36,7 +42,7 @@ import static org.junit.Assert.fail; */ public class RealNodeRepositoryTest { private JDisc container; - private ConfigServerApiImpl configServerApi; + private NodeRepository nodeRepositoryApi; private int findRandomOpenPort() throws IOException { @@ -63,9 +69,10 @@ public class RealNodeRepositoryTest { try { final int port = findRandomOpenPort(); container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable); - configServerApi = ConfigServerApiImpl.createWithSocketFactory( + ConfigServerApi configServerApi = ConfigServerApiImpl.createWithSocketFactory( Collections.singletonList(URI.create("http://127.0.0.1:" + port)), SSLConnectionSocketFactory.getSocketFactory()); + waitForJdiscContainerToServe(configServerApi); return; } catch (RuntimeException e) { lastException = e; @@ -74,9 +81,9 @@ public class RealNodeRepositoryTest { throw new RuntimeException("Failed to bind a port in three attempts, giving up", lastException); } - private void waitForJdiscContainerToServe() throws InterruptedException { + private void waitForJdiscContainerToServe(ConfigServerApi configServerApi) throws InterruptedException { Instant start = Instant.now(); - NodeRepository nodeRepositoryApi = new RealNodeRepository(configServerApi); + nodeRepositoryApi = new RealNodeRepository(configServerApi); while (Instant.now().minusSeconds(120).isBefore(start)) { try { nodeRepositoryApi.getNodes("foobar"); @@ -97,8 +104,6 @@ public class RealNodeRepositoryTest { @Test public void testGetContainersToRunApi() throws InterruptedException { - waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new RealNodeRepository(configServerApi); String dockerHostHostname = "dockerhost1.yahoo.com"; final List containersToRun = nodeRepositoryApi.getNodes(dockerHostHostname); @@ -115,9 +120,7 @@ public class RealNodeRepositoryTest { } @Test - public void testGetContainer() throws InterruptedException, IOException { - waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new RealNodeRepository(configServerApi); + public void testGetContainer() { String hostname = "host4.yahoo.com"; Optional node = nodeRepositoryApi.getNode(hostname); assertThat(node.isPresent(), is(true)); @@ -125,18 +128,14 @@ public class RealNodeRepositoryTest { } @Test - public void testGetContainerForNonExistingNode() throws InterruptedException, IOException { - waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new RealNodeRepository(configServerApi); + public void testGetContainerForNonExistingNode() { String hostname = "host-that-does-not-exist"; Optional node = nodeRepositoryApi.getNode(hostname); assertFalse(node.isPresent()); } @Test - public void testUpdateNodeAttributes() throws InterruptedException, IOException { - waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new RealNodeRepository(configServerApi); + public void testUpdateNodeAttributes() { String hostname = "host4.yahoo.com"; nodeRepositoryApi.updateNodeAttributes( hostname, @@ -147,9 +146,7 @@ public class RealNodeRepositoryTest { } @Test(expected = RuntimeException.class) - public void testUpdateNodeAttributesWithBadValue() throws InterruptedException, IOException { - waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new RealNodeRepository(configServerApi); + public void testUpdateNodeAttributesWithBadValue() { String hostname = "host4.yahoo.com"; nodeRepositoryApi.updateNodeAttributes( hostname, @@ -160,10 +157,7 @@ public class RealNodeRepositoryTest { } @Test - public void testMarkAsReady() throws InterruptedException, IOException { - NodeRepository nodeRepositoryApi = new RealNodeRepository(configServerApi); - waitForJdiscContainerToServe(); - + public void testMarkAsReady() { nodeRepositoryApi.setNodeState("host5.yahoo.com", Node.State.dirty); nodeRepositoryApi.setNodeState("host5.yahoo.com", Node.State.ready); @@ -181,4 +175,26 @@ public class RealNodeRepositoryTest { // expected } } + + @Test + public void testAddNodes() { + AddNode host = new AddNode("host123.domain.tld", "default", NodeType.confighost, + Collections.singleton("::1"), new HashSet<>(Arrays.asList("::2", "::3"))); + + AddNode node = new AddNode("host123-1.domain.tld", "host123.domain.tld", "docker", NodeType.config, + new HashSet<>(Arrays.asList("::2", "::3"))); + + List nodesToAdd = Arrays.asList(host, node); + + assertFalse(nodeRepositoryApi.getNode("host123.domain.tld").isPresent()); + nodeRepositoryApi.addNodes(nodesToAdd); + + NodeSpec hostSpecInNodeRepo = nodeRepositoryApi.getNode("host123.domain.tld") + .orElseThrow(RuntimeException::new); + + assertEquals(host.nodeFlavor, hostSpecInNodeRepo.nodeFlavor); + assertEquals(host.nodeType, hostSpecInNodeRepo.nodeType); + + assertTrue(nodeRepositoryApi.getNode("host123-1.domain.tld").isPresent()); + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index cf50a7d8d75..ca68229a896 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.node.admin.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.AddNode; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.Acl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; @@ -16,7 +16,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * Mock with some simple logic @@ -35,6 +34,9 @@ public class NodeRepoMock implements NodeRepository { this.callOrderVerifier = callOrderVerifier; } + @Override + public void addNodes(List nodes) { } + @Override public List getNodes(String baseHostName) { synchronized (monitor) { -- cgit v1.2.3