Merge pull request #23797 from vespa-engine/mpolden/ports-acl-api

Limit trusted node types and ports on config server

15 files changed, 332 insertions, 107 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java
index 0b226d5acc7..e61f9b79d75 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java
@@ -11,48 +11,34 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancers;
 
 import java.util.Comparator;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.stream.StreamSupport;
 
 /**
  * A node ACL declares which nodes, networks and ports a node should trust.
  *
  * @author mpolden
  */
-public class NodeAcl {
-
-    private final Node node;
-    private final Set<Node> trustedNodes;
-    private final Set<String> trustedNetworks;
-    private final Set<Integer> trustedPorts;
-
-    private NodeAcl(Node node, Set<Node> trustedNodes, Set<String> trustedNetworks, Set<Integer> trustedPorts) {
-        this.node = Objects.requireNonNull(node, "node must be non-null");
-        this.trustedNodes = ImmutableSet.copyOf(Objects.requireNonNull(trustedNodes, "trustedNodes must be non-null"));
-        this.trustedNetworks = ImmutableSet.copyOf(Objects.requireNonNull(trustedNetworks, "trustedNetworks must be non-null"));
-        this.trustedPorts = ImmutableSet.copyOf(Objects.requireNonNull(trustedPorts, "trustedPorts must be non-null"));
-    }
-
-    public Node node() {
-        return node;
-    }
-
-    public Set<Node> trustedNodes() {
-        return trustedNodes;
-    }
-
-    public Set<String> trustedNetworks() {
-        return trustedNetworks;
-    }
-
-    public Set<Integer> trustedPorts() {
-        return trustedPorts;
+public record NodeAcl(Node node,
+                      Set<TrustedNode> trustedNodes,
+                      Set<String> trustedNetworks,
+                      Set<Integer> trustedPorts) {
+
+    private static final Set<Integer> RPC_PORTS = Set.of(19070);
+
+    public NodeAcl {
+        Objects.requireNonNull(node, "node must be non-null");
+        ImmutableSet.copyOf(Objects.requireNonNull(trustedNodes, "trustedNodes must be non-null"));
+        ImmutableSet.copyOf(Objects.requireNonNull(trustedNetworks, "trustedNetworks must be non-null"));
+        ImmutableSet.copyOf(Objects.requireNonNull(trustedPorts, "trustedPorts must be non-null"));
     }
 
     public static NodeAcl from(Node node, NodeList allNodes, LoadBalancers loadBalancers) {
-        Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname));
+        Set<TrustedNode> trustedNodes = new TreeSet<>(Comparator.comparing(TrustedNode::hostname));
         Set<Integer> trustedPorts = new LinkedHashSet<>();
         Set<String> trustedNetworks = new LinkedHashSet<>();
 
@@ -65,9 +51,9 @@ public class NodeAcl {
         // - nodes in same application
         // - load balancers allocated to application
         trustedPorts.add(22);
-        allNodes.parentOf(node).ifPresent(trustedNodes::add);
+        allNodes.parentOf(node).map(TrustedNode::of).ifPresent(trustedNodes::add);
         node.allocation().ifPresent(allocation -> {
-            trustedNodes.addAll(allNodes.owner(allocation.owner()).asList());
+            trustedNodes.addAll(TrustedNode.of(allNodes.owner(allocation.owner())));
             loadBalancers.list(allocation.owner()).asList()
                          .stream()
                          .map(LoadBalancer::instance)
@@ -77,57 +63,77 @@ public class NodeAcl {
         });
 
         switch (node.type()) {
-            case tenant:
+            case tenant -> {
                 // Tenant nodes in other states than ready, trust:
                 // - config servers
                 // - proxy nodes
                 // - parents of the nodes in the same application: If some nodes are on a different IP version
                 //   or only a subset of them are dual-stacked, the communication between the nodes may be NAT-ed
                 //   via parent's IP address
-                trustedNodes.addAll(allNodes.nodeType(NodeType.config).asList());
-                trustedNodes.addAll(allNodes.nodeType(NodeType.proxy).asList());
-                node.allocation().ifPresent(allocation ->
-                                                    trustedNodes.addAll(allNodes.parentsOf(allNodes.owner(allocation.owner())).asList()));
-
+                trustedNodes.addAll(TrustedNode.of(allNodes.nodeType(NodeType.config)));
+                trustedNodes.addAll(TrustedNode.of(allNodes.nodeType(NodeType.proxy)));
+                node.allocation().ifPresent(allocation -> trustedNodes.addAll(TrustedNode.of(allNodes.parentsOf(allNodes.owner(allocation.owner())))));
                 if (node.state() == Node.State.ready) {
                     // Tenant nodes in state ready, trust:
                     // - All tenant nodes in zone. When a ready node is allocated to an application there's a brief
                     //   window where current ACLs have not yet been applied on the node. To avoid service disruption
                     //   during this window, ready tenant nodes trust all other tenant nodes
-                    trustedNodes.addAll(allNodes.nodeType(NodeType.tenant).asList());
+                    trustedNodes.addAll(TrustedNode.of(allNodes.nodeType(NodeType.tenant)));
                 }
-                break;
-
-            case config:
+            }
+            case config -> {
                 // Config servers trust:
-                // - all nodes
+                // - port 19070 (RPC) from all tenant nodes (and their hosts, in case traffic is NAT-ed via parent)
+                // - port 19070 (RPC) from all proxy nodes (and their hosts, in case traffic is NAT-ed via parent)
                 // - port 4443 from the world
-                trustedNodes.addAll(allNodes.asList());
+                trustedNodes.addAll(TrustedNode.of(allNodes.nodeType(NodeType.host, NodeType.tenant,
+                                                                     NodeType.proxyhost, NodeType.proxy),
+                                                   RPC_PORTS));
                 trustedPorts.add(4443);
-                break;
-
-            case proxy:
+            }
+            case proxy -> {
                 // Proxy nodes trust:
                 // - config servers
                 // - all connections from the world on 443 (production traffic) and 4443 (health checks)
-                trustedNodes.addAll(allNodes.nodeType(NodeType.config).asList());
+                trustedNodes.addAll(TrustedNode.of(allNodes.nodeType(NodeType.config)));
                 trustedPorts.add(443);
                 trustedPorts.add(4443);
-                break;
-
-            case controller:
+            }
+            case controller -> {
                 // Controllers:
                 // - port 4443 (HTTPS + Athenz) from the world
                 // - port 443 (HTTPS + Okta) from the world
                 trustedPorts.add(4443);
                 trustedPorts.add(443);
-                break;
-
-            default:
-                throw new IllegalArgumentException("Don't know how to create ACL for " + node +
-                                                   " of type " + node.type());
+            }
+            default -> throw new IllegalArgumentException("Don't know how to create ACL for " + node +
+                                                          " of type " + node.type());
         }
         return new NodeAcl(node, trustedNodes, trustedNetworks, trustedPorts);
     }
 
+    public record TrustedNode(String hostname, NodeType type, Set<String> ipAddresses, Set<Integer> ports) {
+
+        /** Trust given ports from node */
+        public static TrustedNode of(Node node, Set<Integer> ports) {
+            return new TrustedNode(node.hostname(), node.type(), node.ipConfig().primary(), ports);
+        }
+
+        /** Trust all ports from given node */
+        public static TrustedNode of(Node node) {
+            return of(node, Set.of());
+        }
+
+        public static List<TrustedNode> of(Iterable<Node> nodes, Set<Integer> ports) {
+            return StreamSupport.stream(nodes.spliterator(), false)
+                                .map(node -> TrustedNode.of(node, ports))
+                                .toList();
+        }
+
+        public static List<TrustedNode> of(Iterable<Node> nodes) {
+            return of(nodes, Set.of());
+        }
+
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java
index b03d37b4d46..45987338dae 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java
@@ -47,11 +47,15 @@ public class NodeAclResponse extends SlimeJsonResponse {
     }
 
     private void toSlime(NodeAcl nodeAcl, Cursor array) {
-        nodeAcl.trustedNodes().forEach(node -> node.ipConfig().primary().forEach(ipAddress -> {
+        nodeAcl.trustedNodes().forEach(node -> node.ipAddresses().forEach(ipAddress -> {
             Cursor object = array.addObject();
             object.setString("hostname", node.hostname());
             object.setString("type", node.type().name());
             object.setString("ipAddress", ipAddress);
+            if (!node.ports().isEmpty()) {
+                Cursor portsArray = object.setArray("ports");
+                node.ports().stream().sorted().forEach(portsArray::addLong);
+            }
             object.setString("trustedBy", nodeAcl.node().hostname());
         }));
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 78c89118a78..e09c8a55b22 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -159,6 +159,11 @@ public class MockNodeRepository extends NodeRepository {
         nodes().fail("dockerhost6.yahoo.com", Agent.operator, getClass().getSimpleName());
         nodes().removeRecursively("dockerhost6.yahoo.com");
 
+        // Activate config servers
+        ApplicationId cfgApp = ApplicationId.from("cfg", "cfg", "cfg");
+        ClusterSpec cfgCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("configservers")).vespaVersion("6.42").build();
+        activate(provisioner.prepare(cfgApp, cfgCluster, Capacity.fromRequiredNodeType(NodeType.config), null), cfgApp, provisioner);
+
         ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp"));
         ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build();
         activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), null), zoneApp, provisioner);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
index 2346b9e2fab..ea9c0e1193d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
@@ -10,11 +10,10 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.node.NodeAcl;
+import com.yahoo.vespa.hosted.provision.node.NodeAcl.TrustedNode;
 import org.junit.Test;
 
-import java.util.Collections;
 import java.util.Comparator;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Supplier;
@@ -56,7 +55,7 @@ public class AclProvisioningTest {
         Supplier<NodeAcl> nodeAcls = () -> node.acl(tester.nodeRepository().nodes().list(), tester.nodeRepository().loadBalancers());
 
         // Trusted nodes are active nodes in same application, proxy nodes and config servers
-        assertAcls(List.of(activeNodes, proxyNodes, configServers.asList(), hostOfNode),
+        assertAcls(trustedNodesOf(List.of(activeNodes, proxyNodes, configServers.asList(), hostOfNode)),
                    Set.of("10.2.3.0/24", "10.4.5.0/24"),
                    List.of(nodeAcls.get()));
     }
@@ -78,28 +77,38 @@ public class AclProvisioningTest {
         NodeList tenantNodes = tester.nodeRepository().nodes().list().nodeType(NodeType.tenant);
 
         // Trusted nodes are all proxy-, config-, and, tenant-nodes
-        assertAcls(List.of(proxyNodes, configServers.asList(), tenantNodes.asList()), List.of(nodeAcl));
+        assertAcls(trustedNodesOf(List.of(proxyNodes, configServers.asList(), tenantNodes.asList())), List.of(nodeAcl));
     }
 
     @Test
     public void trusted_nodes_for_config_server() {
-        NodeList configServers = tester.makeConfigServers(3, "default", Version.fromString("6.123.456"));
+        NodeList configNodes = tester.makeConfigServers(3, "default", Version.fromString("6.123.456"));
 
         // Populate repo
-        tester.makeReadyNodes(10, nodeResources);
+        List<Node> proxyHosts = tester.makeReadyNodes(2, nodeResources, NodeType.proxyhost, 5);
         List<Node> proxyNodes = tester.makeReadyNodes(3, "default", NodeType.proxy);
+        tester.makeReadyHosts(2, nodeResources)
+              .activateTenantHosts();
 
-        // Allocate 2 nodes
-        deploy(4);
-        NodeList tenantNodes = tester.nodeRepository().nodes().list().nodeType(NodeType.tenant);
+        // Allocate nodes
+        deploy(2);
+        NodeList nodes = tester.nodeRepository().nodes().list();
+        NodeList tenantNodes = nodes.nodeType(NodeType.tenant);
+        NodeList tenantHosts = nodes.nodeType(NodeType.host);
 
         // Get trusted nodes for the first config server
         Node node = tester.nodeRepository().nodes().node("cfg1")
                 .orElseThrow(() -> new RuntimeException("Failed to find cfg1"));
-        NodeAcl nodeAcl = node.acl(tester.nodeRepository().nodes().list(), tester.nodeRepository().loadBalancers());
-
-        // Trusted nodes is all tenant nodes, all proxy nodes, all config servers and load balancer subnets
-        assertAcls(List.of(tenantNodes.asList(), proxyNodes, configServers.asList()), Set.of("10.2.3.0/24", "10.4.5.0/24"), List.of(nodeAcl));
+        NodeAcl nodeAcl = node.acl(nodes, tester.nodeRepository().loadBalancers());
+
+        // Trusted nodes is all tenant nodes+hosts, all proxy nodes+hosts, all config servers and load balancer subnets
+        assertAcls(List.of(TrustedNode.of(tenantHosts, Set.of(19070)),
+                           TrustedNode.of(tenantNodes, Set.of(19070)),
+                           TrustedNode.of(proxyHosts, Set.of(19070)),
+                           TrustedNode.of(proxyNodes, Set.of(19070)),
+                           TrustedNode.of(configNodes)),
+                   Set.of("10.2.3.0/24", "10.4.5.0/24"),
+                   List.of(nodeAcl));
         assertEquals(Set.of(22, 4443), nodeAcl.trustedPorts());
     }
 
@@ -121,7 +130,7 @@ public class AclProvisioningTest {
         NodeAcl nodeAcl = node.acl(tester.nodeRepository().nodes().list(), tester.nodeRepository().loadBalancers());
 
         // Trusted nodes is all config servers and all proxy nodes
-        assertAcls(List.of(proxyNodes.asList(), configServers.asList()), List.of(nodeAcl));
+        assertAcls(trustedNodesOf(List.of(proxyNodes.asList(), configServers.asList())), List.of(nodeAcl));
         assertEquals(Set.of(22, 443, 4443), nodeAcl.trustedPorts());
     }
 
@@ -146,7 +155,7 @@ public class AclProvisioningTest {
                     .findFirst()
                     .orElseThrow(() -> new RuntimeException("Expected to find ACL for node " + node.hostname()));
             assertEquals(host.hostname(), node.parentHostname().get());
-            assertAcls(List.of(configServers.asList(), nodes, List.of(host)), nodeAcl);
+            assertAcls(trustedNodesOf(List.of(configServers.asList(), nodes, List.of(host))), nodeAcl);
         }
     }
 
@@ -160,7 +169,7 @@ public class AclProvisioningTest {
 
         // Controllers and hosts all trust each other
         NodeAcl controllerAcl = controllers.get(0).acl(tester.nodeRepository().nodes().list(), tester.nodeRepository().loadBalancers());
-        assertAcls(List.of(controllers), Set.of("10.2.3.0/24", "10.4.5.0/24"), List.of(controllerAcl));
+        assertAcls(trustedNodesOf(List.of(controllers)), Set.of("10.2.3.0/24", "10.4.5.0/24"), List.of(controllerAcl));
         assertEquals(Set.of(22, 4443, 443), controllerAcl.trustedPorts());
     }
 
@@ -203,10 +212,16 @@ public class AclProvisioningTest {
         NodeAcl nodeAcl = readyNodes.get(0).acl(tester.nodeRepository().nodes().list(), tester.nodeRepository().loadBalancers());
 
         assertEquals(3, nodeAcl.trustedNodes().size());
-        Iterator<Node> trustedNodes = nodeAcl.trustedNodes().iterator();
-        assertEquals(Set.of("127.0.1.1"), trustedNodes.next().ipConfig().primary());
-        assertEquals(Set.of("127.0.1.2"), trustedNodes.next().ipConfig().primary());
-        assertEquals(Set.of("127.0.1.3"), trustedNodes.next().ipConfig().primary());
+        assertEquals(List.of(Set.of("127.0.1.1"), Set.of("127.0.1.2"), Set.of("127.0.1.3")),
+                     nodeAcl.trustedNodes().stream().map(TrustedNode::ipAddresses).toList());
+    }
+
+    private static List<List<TrustedNode>> trustedNodesOf(List<List<Node>> nodes, Set<Integer> ports) {
+        return nodes.stream().map(node -> TrustedNode.of(node, ports)).toList();
+    }
+
+    private static List<List<TrustedNode>> trustedNodesOf(List<List<Node>> nodes) {
+        return trustedNodesOf(nodes, Set.of());
     }
 
     private List<Node> deploy(int nodeCount) {
@@ -217,24 +232,24 @@ public class AclProvisioningTest {
         return tester.deploy(application, Capacity.from(new ClusterResources(nodeCount, 1, nodeResources)));
     }
 
-    private static void assertAcls(List<List<Node>> expected, NodeAcl actual) {
-        assertAcls(expected, Collections.singletonList(actual));
+    private static void assertAcls(List<List<TrustedNode>> expected, NodeAcl actual) {
+        assertAcls(expected, List.of(actual));
     }
 
-    private static void assertAcls(List<List<Node>> expectedNodes, List<NodeAcl> actual) {
+    private static void assertAcls(List<List<TrustedNode>> expectedNodes, List<NodeAcl> actual) {
         assertAcls(expectedNodes, Set.of(), actual);
     }
 
-    private static void assertAcls(List<List<Node>> expectedNodes, Set<String> expectedNetworks, List<NodeAcl> actual) {
-        List<Node> expectedTrustedNodes = expectedNodes.stream()
+    private static void assertAcls(List<List<TrustedNode>> expectedNodes, Set<String> expectedNetworks, List<NodeAcl> actual) {
+        List<TrustedNode> expectedTrustedNodes = expectedNodes.stream()
                 .flatMap(List::stream)
                 .distinct()
-                .sorted(Comparator.comparing(Node::hostname))
+                .sorted(Comparator.comparing(TrustedNode::hostname))
                 .collect(Collectors.toList());
-        List<Node> actualTrustedNodes = actual.stream()
+        List<TrustedNode> actualTrustedNodes = actual.stream()
                 .flatMap(acl -> acl.trustedNodes().stream())
                 .distinct()
-                .sorted(Comparator.comparing(Node::hostname))
+                .sorted(Comparator.comparing(TrustedNode::hostname))
                 .collect(Collectors.toList());
         assertEquals(expectedTrustedNodes, actualTrustedNodes);
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java
index 19af4d00e54..50ddf9c6672 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java
@@ -75,13 +75,13 @@ public class NodesV2ApiTest {
                          new byte[0], Request.Method.POST));
         assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2",
                          new byte[0], Request.Method.POST));
-        assertRestart(13, new Request("http://localhost:8080/nodes/v2/command/restart",
+        assertRestart(15, new Request("http://localhost:8080/nodes/v2/command/restart",
                          new byte[0], Request.Method.POST));
         tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"),
                                      "\"restartGeneration\":3");
 
         // POST reboot command
-        assertReboot(14, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active",
+        assertReboot(16, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active",
                         new byte[0], Request.Method.POST));
         assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2",
                         new byte[0], Request.Method.POST));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json
index 55891309856..4223a1b186a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json
@@ -28,202 +28,244 @@
       "hostname": "dockerhost1.yahoo.com",
       "type": "host",
       "ipAddress": "127.0.100.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost1.yahoo.com",
       "type": "host",
       "ipAddress": "::100:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost2.yahoo.com",
       "type": "host",
       "ipAddress": "127.0.101.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost2.yahoo.com",
       "type": "host",
       "ipAddress": "::101:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost3.yahoo.com",
       "type": "host",
       "ipAddress": "127.0.102.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost3.yahoo.com",
       "type": "host",
       "ipAddress": "::102:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost4.yahoo.com",
       "type": "host",
       "ipAddress": "127.0.103.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost4.yahoo.com",
       "type": "host",
       "ipAddress": "::103:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost5.yahoo.com",
       "type": "host",
       "ipAddress": "127.0.104.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "dockerhost5.yahoo.com",
       "type": "host",
       "ipAddress": "::104:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host1.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.1.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host1.yahoo.com",
       "type": "tenant",
       "ipAddress": "::1:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host10.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.10.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host10.yahoo.com",
       "type": "tenant",
       "ipAddress": "::10:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host13.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.13.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host13.yahoo.com",
       "type": "tenant",
       "ipAddress": "::13:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host14.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.14.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host14.yahoo.com",
       "type": "tenant",
       "ipAddress": "::14:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host2.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.2.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host2.yahoo.com",
       "type": "tenant",
       "ipAddress": "::2:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host3.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.3.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host3.yahoo.com",
       "type": "tenant",
       "ipAddress": "::3:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host4.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.4.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host4.yahoo.com",
       "type": "tenant",
       "ipAddress": "::4:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host5.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.5.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host5.yahoo.com",
       "type": "tenant",
       "ipAddress": "::5:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host55.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.55.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host55.yahoo.com",
       "type": "tenant",
       "ipAddress": "::55:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host6.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.6.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host6.yahoo.com",
       "type": "tenant",
       "ipAddress": "::6:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host7.yahoo.com",
       "type": "tenant",
       "ipAddress": "127.0.7.1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "host7.yahoo.com",
       "type": "tenant",
       "ipAddress": "::7:1",
+      "ports": [19070],
       "trustedBy": "cfg1.yahoo.com"
     },
     {
       "hostname": "test-node-pool-102-2",
       "type": "tenant",
       "ipAddress": "::102:2",
+      "ports": [19070],
+      "trustedBy": "cfg1.yahoo.com"
+    }
+  ],
+  "trustedNetworks": [
+    {
+      "network": "10.2.3.0/24",
+      "trustedBy": "cfg1.yahoo.com"
+    },
+    {
+      "network": "10.4.5.0/24",
       "trustedBy": "cfg1.yahoo.com"
     }
   ],
-  "trustedNetworks": [],
   "trustedPorts": [
     {
       "port":22,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json
index 61ccea7dc94..c46bc6acbd2 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json
@@ -12,6 +12,8 @@
     @include(docker-node4.json),
     @include(docker-node5.json),
     @include(docker-node2.json),
-    @include(docker-node1.json)
+    @include(docker-node1.json),
+    @include(cfg1.json),
+    @include(cfg2.json)
   ]
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json
index 6e2a3b87fe3..a9708f54d02 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json
@@ -1,14 +1,55 @@
 {
   "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com",
   "id": "cfg1",
-  "state": "ready",
+  "state": "active",
   "type": "config",
   "hostname": "cfg1.yahoo.com",
   "flavor": "default",
   "cpuCores": 2.0,
-  "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote","architecture":"x86_64"},
-  "realResources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote","architecture":"x86_64"},
+  "resources": {
+    "vcpu": 2.0,
+    "memoryGb": 16.0,
+    "diskGb": 400.0,
+    "bandwidthGbps": 10.0,
+    "diskSpeed": "fast",
+    "storageType": "remote",
+    "architecture": "x86_64"
+  },
+  "realResources": {
+    "vcpu": 2.0,
+    "memoryGb": 16.0,
+    "diskGb": 400.0,
+    "bandwidthGbps": 10.0,
+    "diskSpeed": "fast",
+    "storageType": "remote",
+    "architecture": "x86_64"
+  },
   "environment": "BARE_METAL",
+  "owner": {
+    "tenant": "cfg",
+    "application": "cfg",
+    "instance": "cfg"
+  },
+  "membership": {
+    "clustertype": "container",
+    "clusterid": "configservers",
+    "group": "0",
+    "index": 0,
+    "retired": false
+  },
+  "restartGeneration": 0,
+  "currentRestartGeneration": 0,
+  "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+  "wantedVespaVersion": "6.42.0",
+  "requestedResources": {
+    "vcpu": 2.0,
+    "memoryGb": 16.0,
+    "diskGb": 400.0,
+    "bandwidthGbps": 10.0,
+    "diskSpeed": "fast",
+    "storageType": "remote",
+    "architecture": "x86_64"
+  },
   "rebootGeneration": 0,
   "currentRebootGeneration": 0,
   "failCount": 0,
@@ -27,6 +68,16 @@
       "event": "readied",
       "at": 123,
       "agent": "system"
+    },
+    {
+      "event": "reserved",
+      "at": 123,
+      "agent": "application"
+    },
+    {
+      "event": "activated",
+      "at": 123,
+      "agent": "application"
     }
   ],
   "log": [
@@ -44,6 +95,21 @@
       "event": "readied",
       "at": 123,
       "agent": "system"
+    },
+    {
+      "event": "reserved",
+      "at": 123,
+      "agent": "application"
+    },
+    {
+      "event": "reserved",
+      "at": 123,
+      "agent": "application"
+    },
+    {
+      "event": "activated",
+      "at": 123,
+      "agent": "application"
     }
   ],
   "ipAddresses": [
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json
index 20809fe51c3..3bd45acb856 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json
@@ -1,14 +1,55 @@
 {
   "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com",
   "id": "cfg2",
-  "state": "ready",
+  "state": "active",
   "type": "config",
   "hostname": "cfg2.yahoo.com",
   "flavor": "default",
   "cpuCores": 2.0,
-  "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote","architecture":"x86_64"},
-  "realResources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote","architecture":"x86_64"},
+  "resources": {
+    "vcpu": 2.0,
+    "memoryGb": 16.0,
+    "diskGb": 400.0,
+    "bandwidthGbps": 10.0,
+    "diskSpeed": "fast",
+    "storageType": "remote",
+    "architecture": "x86_64"
+  },
+  "realResources": {
+    "vcpu": 2.0,
+    "memoryGb": 16.0,
+    "diskGb": 400.0,
+    "bandwidthGbps": 10.0,
+    "diskSpeed": "fast",
+    "storageType": "remote",
+    "architecture": "x86_64"
+  },
   "environment": "BARE_METAL",
+  "owner": {
+    "tenant": "cfg",
+    "application": "cfg",
+    "instance": "cfg"
+  },
+  "membership": {
+    "clustertype": "container",
+    "clusterid": "configservers",
+    "group": "0",
+    "index": 1,
+    "retired": false
+  },
+  "restartGeneration": 0,
+  "currentRestartGeneration": 0,
+  "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0",
+  "wantedVespaVersion": "6.42.0",
+  "requestedResources": {
+    "vcpu": 2.0,
+    "memoryGb": 16.0,
+    "diskGb": 400.0,
+    "bandwidthGbps": 10.0,
+    "diskSpeed": "fast",
+    "storageType": "remote",
+    "architecture": "x86_64"
+  },
   "rebootGeneration": 0,
   "currentRebootGeneration": 0,
   "failCount": 0,
@@ -27,6 +68,16 @@
       "event": "readied",
       "at": 123,
       "agent": "system"
+    },
+    {
+      "event": "reserved",
+      "at": 123,
+      "agent": "application"
+    },
+    {
+      "event": "activated",
+      "at": 123,
+      "agent": "application"
     }
   ],
   "log": [
@@ -44,6 +95,21 @@
       "event": "readied",
       "at": 123,
       "agent": "system"
+    },
+    {
+      "event": "reserved",
+      "at": 123,
+      "agent": "application"
+    },
+    {
+      "event": "reserved",
+      "at": 123,
+      "agent": "application"
+    },
+    {
+      "event": "activated",
+      "at": 123,
+      "agent": "application"
     }
   ],
   "ipAddresses": [
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json
index 82e31abfcfa..7ae283e3916 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json
@@ -31,6 +31,25 @@
       ]
     },
     {
+      "id": "cfg:cfg:cfg:configservers",
+      "state": "reserved",
+      "changedAt": 123,
+      "application": "cfg",
+      "tenant": "cfg",
+      "instance": "cfg",
+      "cluster": "configservers",
+      "hostname": "lb-cfg.cfg.cfg-configservers",
+      "dnsZone": "zone-id-1",
+      "networks": [
+        "10.2.3.0/24",
+        "10.4.5.0/24"
+      ],
+      "ports": [
+        4443
+      ],
+      "reals": []
+    },
+    {
       "id": "tenant4:application4:instance4:id4",
       "state": "active",
       "changedAt": 123,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json
index 2b650bad39b..66b44726e7e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json
@@ -1,10 +1,10 @@
 {
   "nodes": [
     @include(node7.json),
-    @include(cfg1.json),
     @include(node3.json),
-    @include(cfg2.json),
     @include(node10.json),
+    @include(cfg1.json),
+    @include(cfg2.json),
     @include(docker-node3.json),
     @include(node14.json),
     @include(node4.json),
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json
index 55e216f454a..7b52bc576ae 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json
@@ -1,10 +1,10 @@
 {
   "nodes": [
     @include(node7.json),
-    @include(cfg1.json),
     @include(node3.json),
-    @include(cfg2.json),
     @include(node10.json),
+    @include(cfg1.json),
+    @include(cfg2.json),
     @include(docker-node3.json),
     @include(node14.json),
     @include(node4.json),
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json
index 54ff2bc232f..86da5fb6e62 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json
@@ -4,16 +4,16 @@
       "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com"
     },
     {
-      "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com"
+      "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com"
     },
     {
-      "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com"
+      "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com"
     },
     {
-      "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com"
+      "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com"
     },
     {
-      "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com"
+      "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com"
     },
     {
       "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com"
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json
index 27767be6315..5ece0e642f1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json
@@ -9,9 +9,7 @@
     "ready": {
       "url": "http://localhost:8080/nodes/v2/state/ready",
       "nodes": [
-        @include(node3.json),
-        @include(cfg1.json),
-        @include(cfg2.json)
+        @include(node3.json)
       ]
     },
     "reserved": {
@@ -34,7 +32,9 @@
         @include(docker-node4.json),
         @include(docker-node5.json),
         @include(docker-node2.json),
-        @include(docker-node1.json)
+        @include(docker-node1.json),
+        @include(cfg1.json),
+        @include(cfg2.json)
       ]
     },
     "inactive": {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json
index 38505726def..788eb6d359f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json
@@ -1,6 +1,6 @@
 {
   "totalCost": 8.591999999999999,
-  "totalAllocatedCost": 5.356,
+  "totalAllocatedCost": 6.468,
   "load": {
     "cpu": 0.0,
     "memory": 0.0,