Allow applications to request exlcusive access to hosts

6 files changed, 106 insertions, 121 deletions

diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
index 9b1fb45a548..e061c5d07ee 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
@@ -6,7 +6,7 @@ import java.util.Optional;
 /**
  * A capacity request.
  *
- * @author lulf
+ * @author Ulf Lilleengen
  * @author bratseth
  */
 public final class Capacity {
@@ -16,28 +16,28 @@ public final class Capacity {
     private final boolean required;
 
     private final Optional<String> flavor;
-    
+
     private final NodeType type;
 
-    private Capacity(int nodeCount, boolean required, Optional<String> flavor, NodeType type) {
+    private Capacity(int nodeCount, Optional<String> flavor, boolean required, NodeType type) {
         this.nodeCount = nodeCount;
-        this.flavor = flavor;
         this.required = required;
+        this.flavor = flavor;
         this.type = type;
     }
 
     /** Returns the number of nodes requested */
     public int nodeCount() { return nodeCount; }
 
-    /** Returns whether the requested number of nodes must be met exactly for a request for this to succeed */
-    public boolean isRequired() { return required; }
-
     /**
      * The node flavor requested, or empty if no particular flavor is specified.
      * This may be satisfied by the requested flavor or a suitable replacement
      */
     public Optional<String> flavor() { return flavor; }
 
+    /** Returns whether the requested number of nodes must be met exactly for a request for this to succeed */
+    public boolean isRequired() { return required; }
+
     /**
      * Returns the node type (role) requested. This is tenant nodes by default.
      * If some other type is requested the node count and flavor may be ignored
@@ -52,37 +52,16 @@ public final class Capacity {
 
     /** Creates this from a desired node count: The request may be satisfied with a smaller number of nodes. */
     public static Capacity fromNodeCount(int capacity) {
-        return fromNodeCount(capacity, Optional.empty());
-    }
-    /** Creates this from a desired node count: The request may be satisfied with a smaller number of nodes. */
-    public static Capacity fromNodeCount(int nodeCount, String flavor) {
-        return fromNodeCount(nodeCount, Optional.of(flavor));
-    }
-    /** Creates this from a desired node count: The request may be satisfied with a smaller number of nodes. */
-    public static Capacity fromNodeCount(int nodeCount, Optional<String> flavor) {
-        return new Capacity(nodeCount, false, flavor, NodeType.tenant);
-    }
-
-    /** Creates this from a required node count: Requests must fail unless the node count can be satisfied exactly */
-    public static Capacity fromRequiredNodeCount(int nodeCount) {
-        return fromRequiredNodeCount(nodeCount, Optional.empty());
-    }
-    /** Creates this from a required node count: Requests must fail unless the node count can be satisfied exactly */
-    public static Capacity fromRequiredNodeCount(int nodeCount, String flavor) {
-        return fromRequiredNodeCount(nodeCount, Optional.of(flavor));
-    }
-    /** Creates this from a required node count: Requests must fail unless the node count can be satisfied exactly */
-    public static Capacity fromRequiredNodeCount(int nodeCount, Optional<String> flavor) {
-        return new Capacity(nodeCount, true, flavor, NodeType.tenant);
+        return fromNodeCount(capacity, Optional.empty(), false);
     }
 
     public static Capacity fromNodeCount(int nodeCount, Optional<String> flavor, boolean required) {
-        return new Capacity(nodeCount, required, flavor, NodeType.tenant);
+        return new Capacity(nodeCount, flavor, required, NodeType.tenant);
     }
 
     /** Creates this from a node type */
     public static Capacity fromRequiredNodeType(NodeType type) {
-        return new Capacity(0, true, Optional.empty(), type);
+        return new Capacity(0,  Optional.empty(), true, type);
     }
 
 }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
index 3f0da396431..6c08a796c20 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
@@ -4,8 +4,8 @@ package com.yahoo.config.provision;
 import com.yahoo.component.Version;
 
 /**
- * A node's membership in a cluster.
- * This is a value object.
+ * A node's membership in a cluster. This is a value object.
+ * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired]
  *
  * @author bratseth
  */
@@ -19,26 +19,20 @@ public class ClusterMembership {
     protected ClusterMembership() {}
 
     private ClusterMembership(String stringValue, Version vespaVersion) {
-        String restValue;
-        if (stringValue.endsWith("/retired")) {
-            retired = true;
-            restValue = stringValue.substring(0, stringValue.length() - "/retired".length());
-        }
-        else {
-            retired = false;
-            restValue = stringValue;
-        }
-
-        String[] components = restValue.split("/");
-
-        if ( components.length == 3) // Aug 2016: This should never happen any more
-            initWithoutGroup(components, vespaVersion);
-        else if (components.length == 4)
-            initWithGroup(components, vespaVersion);
-        else
+        String[] components = stringValue.split("/");
+        if (components.length < 4 || components.length > 6)
             throw new RuntimeException("Could not parse '" + stringValue + "' to a cluster membership. " +
-                                       "Expected 'id/type.index[/group]'");
+                                       "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive]'");
 
+        boolean exclusive = false;
+        if (components.length > 4) {
+            exclusive = components[4].equals("exclusive");
+            retired = components[components.length-1].equals("retired");
+        }
+
+        this.cluster = ClusterSpec.from(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1]),
+                                        ClusterSpec.Group.from(Integer.valueOf(components[2])), vespaVersion, exclusive);
+        this.index = Integer.parseInt(components[3]);
         this.stringValue = toStringValue();
     }
 
@@ -49,23 +43,14 @@ public class ClusterMembership {
         this.stringValue = toStringValue();
     }
 
-    private void initWithoutGroup(String[] components, Version vespaVersion) {
-        this.cluster = ClusterSpec.request(ClusterSpec.Type.valueOf(components[0]),
-                                           ClusterSpec.Id.from(components[1]),
-                                           vespaVersion);
-        this.index = Integer.parseInt(components[2]);
-    }
-
-    private void initWithGroup(String[] components, Version vespaVersion) {
-        this.cluster = ClusterSpec.from(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1]),
-                                        ClusterSpec.Group.from(Integer.valueOf(components[2])), vespaVersion);
-        this.index = Integer.parseInt(components[3]);
-    }
-
     protected String toStringValue() {
-        return cluster.type().name() + "/" + cluster.id().value() +
-                ( cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") + "/" + index +
-                ( retired ? "/retired" : "");
+        return cluster.type().name() +
+               "/" + cluster.id().value() +
+               (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") +
+               "/" + index +
+               ( cluster.isExclusive() ? "/exclusive" : "") +
+               ( retired ? "/retired" : "");
+
     }
 
     /** Returns the cluster this node is a member of */
@@ -87,7 +72,12 @@ public class ClusterMembership {
         return new ClusterMembership(cluster, index, false);
     }
 
+    // TODO: Remove after April 2018
     public ClusterMembership changeCluster(ClusterSpec newCluster) {
+        return with(newCluster);
+    }
+
+    public ClusterMembership with(ClusterSpec newCluster) {
         return new ClusterMembership(newCluster, index, retired);
     }
 
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
index bcc13f2c3e9..a0df2547631 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
@@ -22,11 +22,14 @@ public final class ClusterSpec {
 
     private final Version vespaVersion;
 
-    private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion) {
+    private boolean exclusive;
+
+    private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive) {
         this.type = type;
         this.id = id;
         this.groupId = groupId;
         this.vespaVersion = vespaVersion;
+        this.exclusive = exclusive;
     }
 
     /** Returns the cluster type */
@@ -40,13 +43,40 @@ public final class ClusterSpec {
     /** Returns the group within the cluster this specifies, or empty to specify the whole cluster */
     public Optional<Group> group() { return groupId; }
 
-    public ClusterSpec changeGroup(Optional<Group> newGroup) { return new ClusterSpec(type, id, newGroup, vespaVersion); }
+    /**
+     * Returns whether the physical hosts running the nodes of this application can
+     * also run nodes of other applications. Using exclusive nodes for containers increases security
+     * and increases cost.
+     */
+    public boolean isExclusive() { return exclusive; }
+
+    // TODO: Remove after April 2018
+    public ClusterSpec changeGroup(Optional<Group> newGroup) {
+        return with(newGroup);
+    }
+
+    public ClusterSpec with(Optional<Group> newGroup) {
+        return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive);
+    }
+
+    public ClusterSpec exclusive(boolean exclusive) {
+        return new ClusterSpec(type, id, groupId, vespaVersion, exclusive);
+    }
 
+    // TODO: Remove after April 2018
     public static ClusterSpec request(Type type, Id id, Version vespaVersion) {
-        return new ClusterSpec(type, id, Optional.empty(), vespaVersion);
+        return request(type, id, vespaVersion, false);
+    }
+    public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive) {
+        return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive);
     }
+
+    // TODO: Remove after April 2018
     public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion) {
-        return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion);
+        return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, false);
+    }
+    public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive) {
+        return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive);
     }
 
     @Override
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java
index 675af88596a..a39b8d3ef34 100644
--- a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java
@@ -18,7 +18,7 @@ public class AllocatedHostsTest {
 
     private final HostSpec h1 = new HostSpec("host1", Optional.empty());
     private final HostSpec h2 = new HostSpec("host2", Optional.empty());
-    private final HostSpec h3 = new HostSpec("host3", Optional.of(ClusterMembership.from("container/test/0", com.yahoo.component.Version.fromString("6.73.1"))));
+    private final HostSpec h3 = new HostSpec("host3", Optional.of(ClusterMembership.from("container/test/0/0", com.yahoo.component.Version.fromString("6.73.1"))));
 
     @Test
     public void testAllocatedHostsSerialization() throws IOException {
@@ -37,7 +37,7 @@ public class AllocatedHostsTest {
         assertTrue(serializedAllocatedHosts.getHosts().contains(h2));
         assertTrue(serializedAllocatedHosts.getHosts().contains(h3));
         assertTrue(!getHost(h1.hostname(), serializedAllocatedHosts.getHosts()).membership().isPresent());
-        assertEquals("container/test/0", getHost(h3.hostname(), serializedAllocatedHosts.getHosts()).membership().get().stringValue());
+        assertEquals("container/test/0/0", getHost(h3.hostname(), serializedAllocatedHosts.getHosts()).membership().get().stringValue());
         assertEquals(h3.membership().get().cluster().vespaVersion(), getHost(h3.hostname(), 
                                                                              serializedAllocatedHosts.getHosts()).membership().get().cluster().vespaVersion());
     }
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java
index bc8586d5c52..f1970326137 100644
--- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java
@@ -18,30 +18,20 @@ public class ClusterMembershipTest {
 
     @Test
     public void testContainerServiceInstance() {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"));
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false);
         assertContainerService(ClusterMembership.from(cluster, 3));
     }
 
     @Test
-    public void testContainerServiceInstanceFromString() {
-        assertContainerService(ClusterMembership.from("container/id1/3", Vtag.currentVersion));
-    }
-
-    @Test
     public void testServiceInstance() {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"));
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false);
         assertContentService(ClusterMembership.from(cluster, 37));
     }
 
     @Test
-    public void testServiceInstanceFromString() {
-        assertContentService(ClusterMembership.from("content/id1/37", Vtag.currentVersion));
-    }
-
-    @Test
     public void testServiceInstanceWithGroup() {
         ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"),
-                                               ClusterSpec.Group.from(4), Version.fromString("6.42"));
+                                               ClusterSpec.Group.from(4), Version.fromString("6.42"), false);
         assertContentServiceWithGroup(ClusterMembership.from(cluster, 37));
     }
 
@@ -52,19 +42,14 @@ public class ClusterMembershipTest {
 
     @Test
     public void testServiceInstanceWithRetire() {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"));
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false);
         assertContentServiceWithRetire(ClusterMembership.retiredFrom(cluster, 37));
     }
 
     @Test
-    public void testServiceInstanceWithRetireFromString() {
-        assertContentServiceWithRetire(ClusterMembership.from("content/id1/37/retired", Vtag.currentVersion));
-    }
-
-    @Test
     public void testServiceInstanceWithGroupAndRetire() {
         ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"),
-                                               ClusterSpec.Group.from(4), Version.fromString("6.42"));
+                                               ClusterSpec.Group.from(4), Version.fromString("6.42"), false);
         assertContentServiceWithGroupAndRetire(ClusterMembership.retiredFrom(cluster, 37));
     }
 
@@ -76,7 +61,7 @@ public class ClusterMembershipTest {
     private void assertContainerService(ClusterMembership instance) {
         assertEquals(ClusterSpec.Type.container, instance.cluster().type());
         assertEquals("id1", instance.cluster().id().value());
-        assertEquals(Optional.<ClusterSpec.Group>empty(), instance.cluster().group());
+        assertFalse(instance.cluster().group().isPresent());
         assertEquals(3, instance.index());
         assertEquals("container/id1/3", instance.stringValue());
     }
@@ -84,7 +69,7 @@ public class ClusterMembershipTest {
     private void assertContentService(ClusterMembership instance) {
         assertEquals(ClusterSpec.Type.content, instance.cluster().type());
         assertEquals("id1", instance.cluster().id().value());
-        assertFalse("gr4", instance.cluster().group().isPresent());
+        assertFalse(instance.cluster().group().isPresent());
         assertEquals(37, instance.index());
         assertFalse(instance.retired());
         assertEquals("content/id1/37", instance.stringValue());
@@ -99,6 +84,7 @@ public class ClusterMembershipTest {
         assertEquals("content/id1/4/37", instance.stringValue());
     }
 
+    /** Serializing a spec without a group assigned works, but not deserialization */
     private void assertContentServiceWithRetire(ClusterMembership instance) {
         assertEquals(ClusterSpec.Type.content, instance.cluster().type());
         assertEquals("id1", instance.cluster().id().value());
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java
index adc7105b2a6..e1d6f061446 100644
--- a/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java
@@ -24,25 +24,25 @@ public class HostFilterTest {
         HostFilter type = HostFilter.clusterType(ClusterSpec.Type.content);
         HostFilter id = HostFilter.clusterId(ClusterSpec.Id.from("type1"));
 
-        assertTrue(      all.matches("anyhost", "flavor", membership("container/anytype/0")));
-        assertFalse(hostname.matches("anyhost", "flavor", membership("container/anytype/0")));
-        assertFalse(type.matches("anyhost", "flavor", membership("container/anytype/0")));
-        assertFalse(id.matches("anyhost", "flavor", membership("container/anytype/0")));
+        assertTrue(      all.matches("anyhost", "flavor", membership("container/anytype/0/0")));
+        assertFalse(hostname.matches("anyhost", "flavor", membership("container/anytype/0/0")));
+        assertFalse(type.matches("anyhost", "flavor", membership("container/anytype/0/0")));
+        assertFalse(id.matches("anyhost", "flavor", membership("container/anytype/0/0")));
 
-        assertTrue(      all.matches("anyhost", "flavor", membership("content/anytype/0")));
-        assertFalse(hostname.matches("anyhost", "flavor", membership("content/anytype/0")));
-        assertTrue(     type.matches("anyhost", "flavor", membership("content/anytype/0")));
-        assertFalse(      id.matches("anyhost", "flavor", membership("content/anytype/0")));
+        assertTrue(      all.matches("anyhost", "flavor", membership("content/anytype/0/0")));
+        assertFalse(hostname.matches("anyhost", "flavor", membership("content/anytype/0/0")));
+        assertTrue(     type.matches("anyhost", "flavor", membership("content/anytype/0/0")));
+        assertFalse(      id.matches("anyhost", "flavor", membership("content/anytype/0/0")));
 
-        assertTrue(      all.matches("host1", "flavor", membership("content/anytype/0")));
-        assertTrue( hostname.matches("host1", "flavor", membership("content/anytype/0")));
-        assertTrue(     type.matches("host1", "flavor", membership("content/anytype/0")));
-        assertFalse(      id.matches("host1", "flavor", membership("content/anytype/0")));
+        assertTrue(      all.matches("host1", "flavor", membership("content/anytype/0/0")));
+        assertTrue( hostname.matches("host1", "flavor", membership("content/anytype/0/0")));
+        assertTrue(     type.matches("host1", "flavor", membership("content/anytype/0/0")));
+        assertFalse(      id.matches("host1", "flavor", membership("content/anytype/0/0")));
 
-        assertTrue(      all.matches("host1", "flavor", membership("content/type1/0")));
-        assertTrue( hostname.matches("host1", "flavor", membership("content/type1/0")));
-        assertTrue(     type.matches("host1", "flavor", membership("content/type1/0")));
-        assertTrue(       id.matches("host1", "flavor", membership("content/type1/0")));
+        assertTrue(      all.matches("host1", "flavor", membership("content/type1/0/0")));
+        assertTrue( hostname.matches("host1", "flavor", membership("content/type1/0/0")));
+        assertTrue(     type.matches("host1", "flavor", membership("content/type1/0/0")));
+        assertTrue(       id.matches("host1", "flavor", membership("content/type1/0/0")));
     }
 
     @Test
@@ -52,22 +52,22 @@ public class HostFilterTest {
                                                Collections.singletonList(ClusterSpec.Type.content),
                                                Collections.singletonList(ClusterSpec.Id.from("type1")));
 
-        assertFalse(typeAndId.matches("anyhost", "flavor", membership("content/anyType/0")));
-        assertFalse(typeAndId.matches("anyhost", "flavor", membership("container/type1/0")));
-        assertTrue(typeAndId.matches("anyhost", "flavor", membership("content/type1/0")));
+        assertFalse(typeAndId.matches("anyhost", "flavor", membership("content/anyType/0/0")));
+        assertFalse(typeAndId.matches("anyhost", "flavor", membership("container/type1/0/0")));
+        assertTrue(typeAndId.matches("anyhost", "flavor", membership("content/type1/0/0")));
     }
 
     @Test
     public void testMultiConditionFilterFromStrings() {
         HostFilter typeAndId = HostFilter.from("host1  host2, host3,host4", "  , ,flavor", null, "type1  ");
 
-        assertFalse(typeAndId.matches("anotherhost", "flavor", membership("content/type1/0")));
-        assertTrue(typeAndId.matches("host1", "flavor", membership("content/type1/0")));
-        assertTrue(typeAndId.matches("host2", "flavor", membership("content/type1/0")));
-        assertTrue(typeAndId.matches("host3", "flavor", membership("content/type1/0")));
-        assertTrue(typeAndId.matches("host4", "flavor", membership("content/type1/0")));
-        assertFalse(typeAndId.matches("host1", "flavor", membership("content/type2/0")));
-        assertFalse(typeAndId.matches("host4", "differentflavor", membership("content/type1/0")));
+        assertFalse(typeAndId.matches("anotherhost", "flavor", membership("content/type1/0/0")));
+        assertTrue(typeAndId.matches("host1", "flavor", membership("content/type1/0/0")));
+        assertTrue(typeAndId.matches("host2", "flavor", membership("content/type1/0/0")));
+        assertTrue(typeAndId.matches("host3", "flavor", membership("content/type1/0/0")));
+        assertTrue(typeAndId.matches("host4", "flavor", membership("content/type1/0/0")));
+        assertFalse(typeAndId.matches("host1", "flavor", membership("content/type2/0/0")));
+        assertFalse(typeAndId.matches("host4", "differentflavor", membership("content/type1/0/0")));
     }
 
     private Optional<ClusterMembership> membership(String membershipString) {