Merge pull request #2900 from yahoo/freva/add-trustedNetworks-acl-spec

Freva/add trusted networks acl spec

5 files changed, 91 insertions, 51 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index a61565126ef..fde5669bfd5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -164,8 +165,9 @@ public class NodeRepository extends AbstractComponent {
     /**
      * Returns a set of nodes that should be trusted by the given node.
      */
-    private Set<Node> getTrustedNodes(Node node, NodeList candidates) {
+    private NodeAcl getNodeAcl(Node node, NodeList candidates) {
         Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname));
+        Set<String> trustedNetworks = new HashSet<>();
 
         // For all cases below, trust:
         // - nodes in same application
@@ -196,8 +198,12 @@ public class NodeRepository extends AbstractComponent {
                 break;
 
             case proxy:
+                // No special rules for proxies
+                break;
+
             case host:
-                // No special rules for proxies and Docker hosts
+                // Docker bridge network
+                trustedNetworks.add("172.17.0.0/16");
                 break;
 
             default:
@@ -206,7 +212,7 @@ public class NodeRepository extends AbstractComponent {
                                 node.hostname(), node.type()));
         }
 
-        return Collections.unmodifiableSet(trustedNodes);
+        return new NodeAcl(node, trustedNodes, trustedNetworks);
     }
 
     /**
@@ -217,17 +223,14 @@ public class NodeRepository extends AbstractComponent {
      * @return List of node ACLs
      */
     public List<NodeAcl> getNodeAcls(Node node, boolean children) {
-        List<NodeAcl> nodeAcls = new ArrayList<>();
-
         NodeList candidates = new NodeList(getNodes());
         if (children) {
-            List<Node> childNodes = candidates.childNodes(node).asList();
-            childNodes.forEach(childNode -> nodeAcls.add(new NodeAcl(childNode, getTrustedNodes(childNode, candidates))));
+            return candidates.childNodes(node).asList().stream()
+                    .map(childNode -> getNodeAcl(childNode, candidates))
+                    .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
         } else {
-            nodeAcls.add(new NodeAcl(node, getTrustedNodes(node, candidates)));
+            return Collections.singletonList(getNodeAcl(node, candidates));
         }
-
-        return Collections.unmodifiableList(nodeAcls);
     }
 
     /** Get config node by hostname */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java
index 0386f3a5ddc..a6190f41c07 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java
@@ -7,7 +7,8 @@ import com.yahoo.vespa.hosted.provision.Node;
 import java.util.Set;
 
 /**
- * A node ACL. The ACL contains the node which the ACL is valid for, and a set of nodes that the node should trust.
+ * A node ACL. The ACL contains the node which the ACL is valid for,
+ * a set of nodes and networks that the node should trust.
  *
  * @author mpolden
  */
@@ -15,10 +16,12 @@ public class NodeAcl {
 
     private final Node node;
     private final Set<Node> trustedNodes;
+    private final Set<String> trustedNetworks;
 
-    public NodeAcl(Node node, Set<Node> trustedNodes) {
+    public NodeAcl(Node node, Set<Node> trustedNodes, Set<String> trustedNetworks) {
         this.node = node;
         this.trustedNodes = ImmutableSet.copyOf(trustedNodes);
+        this.trustedNetworks = ImmutableSet.copyOf(trustedNetworks);
     }
 
     public Node node() {
@@ -28,4 +31,8 @@ public class NodeAcl {
     public Set<Node> trustedNodes() {
         return trustedNodes;
     }
+
+    public Set<String> trustedNetworks() {
+        return trustedNetworks;
+    }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java
index 3eb9c0a09f5..2a4f37151de 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java
@@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.List;
 
 /**
  * @author mpolden
@@ -45,16 +44,29 @@ public class NodeAclResponse extends HttpResponse {
         Node node = nodeRepository.getNode(hostname)
                 .orElseGet(() -> nodeRepository.getConfigNode(hostname)
                         .orElseThrow(() -> new NotFoundException("No node with hostname '" + hostname + "'")));
-        toSlime(nodeRepository.getNodeAcls(node, aclsForChildren), object.setArray("trustedNodes"));
+
+        Cursor trustedNodesArray = object.setArray("trustedNodes");
+        nodeRepository.getNodeAcls(node, aclsForChildren).forEach(nodeAcl -> toTrustedNodeSlime(nodeAcl, trustedNodesArray));
+
+        Cursor trustedNetworksArray = object.setArray("trustedNetworks");
+        nodeRepository.getNodeAcls(node, aclsForChildren).forEach(nodeAcl -> toTrustedNetworkSlime(nodeAcl, trustedNetworksArray));
     }
 
-    private void toSlime(List<NodeAcl> nodeAcls, Cursor array) {
-        nodeAcls.forEach(acl -> acl.trustedNodes().forEach(node -> node.ipAddresses().forEach(ipAddress -> {
+    private void toTrustedNodeSlime(NodeAcl nodeAcl, Cursor array) {
+        nodeAcl.trustedNodes().forEach(node -> node.ipAddresses().forEach(ipAddress -> {
             Cursor object = array.addObject();
             object.setString("hostname", node.hostname());
             object.setString("ipAddress", ipAddress);
-            object.setString("trustedBy", acl.node().hostname());
-        })));
+            object.setString("trustedBy", nodeAcl.node().hostname());
+        }));
+    }
+
+    private void toTrustedNetworkSlime(NodeAcl nodeAcl, Cursor array) {
+        nodeAcl.trustedNetworks().forEach(network -> {
+            Cursor object = array.addObject();
+            object.setString("network", network);
+            object.setString("trustedBy", nodeAcl.node().hostname());
+        });
     }
 
     @Override
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
index 3b7a14000ec..134808b7114 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
@@ -22,7 +22,6 @@ import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -40,6 +39,8 @@ public class AclProvisioningTest {
     private ProvisioningTester tester;
     private MockNameResolver nameResolver;
 
+    private final List<String> dockerBridgeNetwork = Collections.singletonList("172.17.0.0/16");
+
     @Before
     public void before() {
         this.curator = new MockCurator();
@@ -146,31 +147,27 @@ public class AclProvisioningTest {
         List<NodeAcl> acls = tester.nodeRepository().getNodeAcls(dockerHostNodes.get(0), false);
 
         // Trusted nodes is all Docker hosts and all config servers
-        assertAcls(Arrays.asList(dockerHostNodes, configServers), acls.get(0));
+        assertAcls(Arrays.asList(dockerHostNodes, configServers), dockerBridgeNetwork, acls.get(0));
     }
 
 
     @Test
-    public void trusted_nodes_for_docker_hosts_and_proxy_nodes_in_zone_application() {
+    public void trusted_nodes_for_docker_hosts_nodes_in_zone_application() {
         ApplicationId applicationId = tester.makeApplicationId(); // use same id for both allocate calls below
         List<Node> configServers = setConfigServers("cfg1:1234,cfg2:1234,cfg3:1234");
 
         // Populate repo
-        tester.makeReadyNodes(3, "default", NodeType.proxy);
         tester.makeReadyNodes(2, "default", NodeType.host);
 
-        // Allocate 3 proxy nodes
-        List<Node> activeProxyNodes = allocateNodes(NodeType.proxy, applicationId);
-        assertEquals(3, activeProxyNodes.size());
-        // Allocate 2 Docker hosts, a total of 5 hosts
-        List<Node> activeDockerHostsAndProxyNodes = allocateNodes(NodeType.host, applicationId);
-        assertEquals(5, activeDockerHostsAndProxyNodes.size());
+        // Allocate 2 Docker hosts
+        List<Node> activeDockerHostNodes = allocateNodes(NodeType.host, applicationId);
+        assertEquals(2, activeDockerHostNodes.size());
 
         // Check trusted nodes for all nodes
-        activeDockerHostsAndProxyNodes.forEach(node -> {
+        activeDockerHostNodes.forEach(node -> {
             System.out.println("Checking node " + node);
             List<NodeAcl> nodeAcls = tester.nodeRepository().getNodeAcls(node, false);
-            assertAcls(Arrays.asList(activeDockerHostsAndProxyNodes, configServers), nodeAcls);
+            assertAcls(Arrays.asList(activeDockerHostNodes, configServers), dockerBridgeNetwork, nodeAcls);
         });
     }
 
@@ -238,17 +235,30 @@ public class AclProvisioningTest {
     }
 
     private static void assertAcls(List<List<Node>> expected, NodeAcl actual) {
-        assertAcls(expected, Collections.singletonList(actual));
+        assertAcls(expected, Collections.emptyList(), Collections.singletonList(actual));
     }
 
     private static void assertAcls(List<List<Node>> expected, List<NodeAcl> actual) {
-        List<Node> nodes = expected.stream()
+        assertAcls(expected, Collections.emptyList(), actual);
+    }
+
+    private static void assertAcls(List<List<Node>> expected, List<String> expectedNetworks, NodeAcl actual) {
+        assertAcls(expected, expectedNetworks, Collections.singletonList(actual));
+    }
+
+    private static void assertAcls(List<List<Node>> expectedNodes, List<String> expectedNetworks, List<NodeAcl> actual) {
+        Set<Node> expectedTrustedNodes = expectedNodes.stream()
                 .flatMap(Collection::stream)
-                .sorted(Comparator.comparing(Node::hostname))
-                .collect(Collectors.toList());
-        List<Node> trustedNodes = actual.stream()
+                .collect(Collectors.toSet());
+        Set<Node> actualTrustedNodes = actual.stream()
                 .flatMap(acl -> acl.trustedNodes().stream())
-                .collect(Collectors.toList());
-        assertEquals(nodes, trustedNodes);
+                .collect(Collectors.toSet());
+        assertEquals(expectedTrustedNodes, actualTrustedNodes);
+
+        Set<String> expectedTrustedNetworks = new HashSet<>(expectedNetworks);
+        Set<String> actualTrustedNetworks = actual.stream()
+                .flatMap(acl -> acl.trustedNetworks().stream())
+                .collect(Collectors.toSet());
+        assertEquals(expectedTrustedNetworks, actualTrustedNetworks);
     }
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index f139f2bc156..c46680b2fe0 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -309,7 +309,7 @@ public class RestApiTest {
                 "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}," +
                 "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}," +
                 "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"foo.yahoo.com\"}" +
-                "]}");
+                "],\"trustedNetworks\":\\[\\]}");
         assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/" + hostname), responsePattern);
     }
 
@@ -319,25 +319,33 @@ public class RestApiTest {
                 "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," +
                 "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," +
                 "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}" +
-                "]}");
+                "],\"trustedNetworks\":\\[\\]}");
         assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/cfg1"), responsePattern);
     }
 
     @Test
+    public void acl_request_by_docker_host() throws Exception {
+        Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[" +
+                "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}," +
+                "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}," +
+                "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"parent1.yahoo.com\"}]," +
+                "\"trustedNetworks\":\\[" +
+                "\\{\"network\":\"172.17.0.0/16\",\"trustedBy\":\"parent1.yahoo.com\"}]}");
+        assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/parent1.yahoo.com"), responsePattern);
+    }
+
+    @Test
     public void acl_response_with_dual_stack_node() throws Exception {
-        assertResponse(new Request("http://localhost:8080/nodes/v2/node",
-                        ("[" + asNodeJson("dual-stack-host.yahoo.com", "default", "127.0.0.1", "::1") + "]").
-                                getBytes(StandardCharsets.UTF_8),
-                        Request.Method.POST),
-                "{\"message\":\"Added 1 nodes to the provisioned state\"}");
         Pattern responsePattern = Pattern.compile("\\{\"trustedNodes\":\\[" +
-                "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," +
-                "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," +
-                "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"cfg1\"}," +
-                "\\{\"hostname\":\"dual-stack-host.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"cfg1\"}," +
-                "\\{\"hostname\":\"dual-stack-host.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"cfg1\"}" +
-                ".*]}");
-        assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/cfg1"), responsePattern);
+                "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\",\"trustedBy\":\"host1.yahoo.com\"}," +
+                "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\",\"trustedBy\":\"host1.yahoo.com\"}," +
+                "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\",\"trustedBy\":\"host1.yahoo.com\"}," +
+                "\\{\"hostname\":\"host1.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"host1.yahoo.com\"}," +
+                "\\{\"hostname\":\"host1.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"host1.yahoo.com\"}," +
+                "\\{\"hostname\":\"host2.yahoo.com\",\"ipAddress\":\"::1\",\"trustedBy\":\"host1.yahoo.com\"}," +
+                "\\{\"hostname\":\"host2.yahoo.com\",\"ipAddress\":\"127.0.0.1\",\"trustedBy\":\"host1.yahoo.com\"}" +
+                "],\"trustedNetworks\":\\[\\]}");
+        assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/host1.yahoo.com"), responsePattern);
     }
 
     @Test