Revert "Bratseth/remove allocation by flavor"

7 files changed, 242 insertions, 57 deletions

diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
index dfa9ab7f6b8..f8535eda44f 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
@@ -41,7 +41,7 @@ public final class Capacity {
     @Deprecated
     public Optional<String> flavor() {
         if (nodeResources().isEmpty()) return Optional.empty();
-        return nodeResources.map(n -> n.toString());
+        return nodeResources.get().legacyName();
     }
 
     /** Returns the resources requested for each node, or empty to leave this decision to provisioning */
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java
index 2bc70efbc15..48c84b8ecb7 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java
@@ -3,14 +3,15 @@ package com.yahoo.config.provision;
 
 import com.yahoo.config.provisioning.FlavorsConfig;
 
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * A host or node flavor.
  * *Host* flavors come from a configured set which corresponds to the actual flavors available in a zone.
- * *Node* flavors are simply a wrapper of a NodeResources object.
+ * *Node* flavors are simply a wrapper of a NodeResources object (for now (May 2019) with the exception of some
+ *        legacy behavior where nodes are allocated by specifying a physical host flavor directly).
  *
  * @author bratseth
  */
@@ -19,8 +20,11 @@ public class Flavor {
     private boolean configured;
     private final String name;
     private final int cost;
+    private final boolean isStock;
     private final Type type;
     private final double bandwidth;
+    private final boolean retired;
+    private List<Flavor> replacesFlavors;
 
     /** The hardware resources of this flavor */
     private NodeResources resources;
@@ -30,22 +34,31 @@ public class Flavor {
         this.configured = true;
         this.name = flavorConfig.name();
         this.cost = flavorConfig.cost();
+        this.isStock = flavorConfig.stock();
         this.type = Type.valueOf(flavorConfig.environment());
         this.resources = new NodeResources(flavorConfig.minCpuCores(),
                                            flavorConfig.minMainMemoryAvailableGb(),
                                            flavorConfig.minDiskAvailableGb(),
                                            flavorConfig.fastDisk() ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow);
         this.bandwidth = flavorConfig.bandwidth();
+        this.retired = flavorConfig.retired();
+        this.replacesFlavors = new ArrayList<>();
     }
 
     /** Creates a *node* flavor from a node resources spec */
     public Flavor(NodeResources resources) {
         Objects.requireNonNull(resources, "Resources cannot be null");
+        if (resources.allocateByLegacyName())
+            throw new IllegalArgumentException("Can not create flavor '" + resources.legacyName() + "' from a flavor: " +
+                                               "Non-docker flavors must be of a configured flavor");
         this.configured = false;
-        this.name = resources.toString();
+        this.name = resources.legacyName().orElse(resources.toString());
         this.cost = 0;
+        this.isStock = true;
         this.type = Type.DOCKER_CONTAINER;
         this.bandwidth = 1;
+        this.retired = false;
+        this.replacesFlavors = List.of();
         this.resources = resources;
     }
 
@@ -60,6 +73,8 @@ public class Flavor {
      */
     public int cost() { return cost; }
     
+    public boolean isStock() { return isStock; }
+
     /**
      * True if this is a configured flavor used for hosts,
      * false if it is a virtual flavor created on the fly from node resources
@@ -78,31 +93,65 @@ public class Flavor {
 
     public double getMinCpuCores() { return resources.vcpu(); }
 
+    /** Returns whether the flavor is retired */
+    public boolean isRetired() {
+        return retired;
+    }
+
     public Type getType() { return type; }
     
     /** Convenience, returns getType() == Type.DOCKER_CONTAINER */
     public boolean isDocker() { return type == Type.DOCKER_CONTAINER; }
 
-    // TODO: Remove after August 2019
-    public String canonicalName() { return name; }
-
-    // TODO: Remove after August 2019
-    public boolean satisfies(Flavor flavor) { return this.equals(flavor); }
-
-    // TODO: Remove after August 2019
-    public boolean isStock() { return false; }
-
-    // TODO: Remove after August 2019
-    public boolean isRetired() { return false; }
+    /**
+     * Returns the canonical name of this flavor - which is the name which should be used as an interface to users.
+     * The canonical name of this flavor is:
+     * <ul>
+     *   <li>If it replaces one flavor, the canonical name of the flavor it replaces
+     *   <li>If it replaces multiple or no flavors - itself
+     * </ul>
+     *
+     * The logic is that we can use this to capture the gritty details of configurations in exact flavor names
+     * but also encourage users to refer to them by a common name by letting such flavor variants declare that they
+     * replace the canonical name we want. However, if a node replaces multiple names, we have no basis for choosing one
+     * of them as the canonical, so we return the current as canonical.
+     */
+    public String canonicalName() {
+        return isCanonical() ? name : replacesFlavors.get(0).canonicalName();
+    }
+    
+    /** Returns whether this is a canonical flavor */
+    public boolean isCanonical() {
+        return replacesFlavors.size() != 1;
+    }
 
-    // TODO: Remove after August 2019
-    public boolean isCanonical() { return false; }
+    /**
+     * The flavors this (directly) replaces.
+     * This is immutable if this is frozen, and a mutable list otherwise.
+     */
+    public List<Flavor> replaces() { return replacesFlavors; }
 
-    // TODO: Remove after August 2019
-    public List<Flavor> replaces() { return Collections.emptyList(); }
+    /**
+     * Returns whether this flavor satisfies the requested flavor, either directly
+     * (by being the same), or by directly or indirectly replacing it
+     */
+    public boolean satisfies(Flavor flavor) {
+        if (this.equals(flavor)) {
+            return true;
+        }
+        if (this.retired) {
+            return false;
+        }
+        for (Flavor replaces : replacesFlavors)
+            if (replaces.satisfies(flavor))
+                return true;
+        return false;
+    }
 
-    // TODO: Remove after August 2019
-    public void freeze() {}
+    /** Irreversibly freezes the content of this */
+    public void freeze() {
+        replacesFlavors = List.copyOf(replacesFlavors);
+    }
 
     @Override
     public int hashCode() { return name.hashCode(); }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java
index a9f031cae70..4d4d3c8cf86 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java
@@ -41,7 +41,10 @@ public class NodeFlavors {
             return Optional.of(configuredFlavors.get(name));
 
         NodeResources nodeResources = NodeResources.fromLegacyName(name);
-        return Optional.of(new Flavor(nodeResources));
+        if (nodeResources.allocateByLegacyName())
+            return Optional.empty();
+        else
+            return Optional.of(new Flavor(nodeResources));
     }
 
     /**
@@ -49,7 +52,8 @@ public class NodeFlavors {
      * and cannot be created on the fly.
      */
     public Flavor getFlavorOrThrow(String flavorName) {
-        return getFlavor(flavorName).orElseThrow(() -> new IllegalArgumentException("Unknown flavor '" + flavorName + "'"));
+        return getFlavor(flavorName).orElseThrow(() -> new IllegalArgumentException("Unknown flavor '" + flavorName +
+                                                                                    "'. Flavors are " + canonicalFlavorNames()));
     }
 
     /** Returns true if this flavor is configured or can be created on the fly */
@@ -57,8 +61,43 @@ public class NodeFlavors {
         return getFlavor(flavorName).isPresent();
     }
 
+    private List<String> canonicalFlavorNames() {
+        return configuredFlavors.values().stream().map(Flavor::canonicalName).distinct().sorted().collect(Collectors.toList());
+    }
+
     private static Collection<Flavor> toFlavors(FlavorsConfig config) {
-        return config.flavor().stream().map(Flavor::new).collect(Collectors.toList());
+        Map<String, Flavor> flavors = new HashMap<>();
+        // First pass, create all flavors, but do not include flavorReplacesConfig.
+        for (FlavorsConfig.Flavor flavorConfig : config.flavor()) {
+            flavors.put(flavorConfig.name(), new Flavor(flavorConfig));
+        }
+        // Second pass, set flavorReplacesConfig to point to correct flavor.
+        for (FlavorsConfig.Flavor flavorConfig : config.flavor()) {
+            Flavor flavor = flavors.get(flavorConfig.name());
+            for (FlavorsConfig.Flavor.Replaces flavorReplacesConfig : flavorConfig.replaces()) {
+                if (! flavors.containsKey(flavorReplacesConfig.name())) {
+                    throw new IllegalStateException("Replaces for " + flavor.name() + 
+                                                    " pointing to a non existing flavor: " + flavorReplacesConfig.name());
+                }
+                flavor.replaces().add(flavors.get(flavorReplacesConfig.name()));
+            }
+            flavor.freeze();
+        }
+        // Third pass, ensure that retired flavors have a replacement
+        for (Flavor flavor : flavors.values()) {
+            if (flavor.isRetired() && !hasReplacement(flavors.values(), flavor)) {
+                throw new IllegalStateException(
+                        String.format("Flavor '%s' is retired, but has no replacement", flavor.name())
+                );
+            }
+        }
+        return flavors.values();
+    }
+
+    private static boolean hasReplacement(Collection<Flavor> flavors, Flavor flavor) {
+        return flavors.stream()
+                .filter(f -> !f.equals(flavor))
+                .anyMatch(f -> f.satisfies(flavor));
     }
 
 }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
index 8ef48f7048f..7e90767c9c5 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
@@ -1,6 +1,7 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.provision;
 
+import java.util.Objects;
 import java.util.Optional;
 
 /**
@@ -21,6 +22,11 @@ public class NodeResources {
     private final double diskGb;
     private final DiskSpeed diskSpeed;
 
+    private final boolean allocateByLegacyName;
+
+    /** The legacy (flavor) name of this, or null if none */
+    private final String legacyName;
+
     /** Create node resources requiring fast disk */
     public NodeResources(double vcpu, double memoryGb, double diskGb) {
         this(vcpu, memoryGb, diskGb, DiskSpeed.fast);
@@ -31,6 +37,18 @@ public class NodeResources {
         this.memoryGb = memoryGb;
         this.diskGb = diskGb;
         this.diskSpeed = diskSpeed;
+        this.allocateByLegacyName = false;
+        this.legacyName = null;
+    }
+
+    private NodeResources(double vcpu, double memoryGb, double diskGb, DiskSpeed diskSpeed,
+                          boolean allocateByLegacyName, String legacyName) {
+        this.vcpu = vcpu;
+        this.memoryGb = memoryGb;
+        this.diskGb = diskGb;
+        this.diskSpeed = diskSpeed;
+        this.allocateByLegacyName = allocateByLegacyName;
+        this.legacyName = legacyName;
     }
 
     public double vcpu() { return vcpu; }
@@ -64,17 +82,24 @@ public class NodeResources {
                                  combine(this.diskSpeed, other.diskSpeed));
     }
 
-    // TODO: Remove after August 2019
+    /**
+     * If this is true, a non-docker legacy name was used to specify this and we'll respect that by mapping directly.
+     * The other getters of this will return 0.
+     */
+    public boolean allocateByLegacyName() { return allocateByLegacyName; }
+
+    /** Returns the legacy name of this, or empty if none. */
     public Optional<String> legacyName() {
-        return Optional.of(toString());
+        return Optional.ofNullable(legacyName);
     }
 
-    // TODO: Remove after August 2019
-    public boolean allocateByLegacyName() { return false; }
-
     private boolean isInterchangeableWith(NodeResources other) {
+        if (this.allocateByLegacyName != other.allocateByLegacyName) return false;
+        if (this.allocateByLegacyName) return legacyName.equals(other.legacyName);
+
         if (this.diskSpeed != DiskSpeed.any && other.diskSpeed != DiskSpeed.any && this.diskSpeed != other.diskSpeed)
             return false;
+
         return true;
     }
 
@@ -90,26 +115,40 @@ public class NodeResources {
         if (o == this) return true;
         if ( ! (o instanceof NodeResources)) return false;
         NodeResources other = (NodeResources)o;
-        if (this.vcpu != other.vcpu) return false;
-        if (this.memoryGb != other.memoryGb) return false;
-        if (this.diskGb != other.diskGb) return false;
-        if (this.diskSpeed != other.diskSpeed) return false;
-        return true;
+        if (allocateByLegacyName) {
+            return this.legacyName.equals(other.legacyName);
+        }
+        else {
+            if (this.vcpu != other.vcpu) return false;
+            if (this.memoryGb != other.memoryGb) return false;
+            if (this.diskGb != other.diskGb) return false;
+            if (this.diskSpeed != other.diskSpeed) return false;
+            return true;
+        }
     }
 
     @Override
     public int hashCode() {
-        return (int)(2503 * vcpu + 22123 * memoryGb + 26987 * diskGb + diskSpeed.hashCode());
+        if (allocateByLegacyName)
+            return legacyName.hashCode();
+        else
+            return (int)(2503 * vcpu + 22123 * memoryGb + 26987 * diskGb + diskSpeed.hashCode());
     }
 
     @Override
     public String toString() {
-        return "[vcpu: " + vcpu + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb" +
-               (diskSpeed != DiskSpeed.fast ? ", disk speed: " + diskSpeed : "") + "]";
+        if (allocateByLegacyName)
+            return "flavor '" + legacyName + "'";
+        else
+            return "[vcpu: " + vcpu + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb" +
+                   (diskSpeed != DiskSpeed.fast ? ", disk speed: " + diskSpeed : "") + "]";
     }
 
     /** Returns true if all the resources of this are the same or larger than the given resources */
     public boolean satisfies(NodeResources other) {
+        if (this.allocateByLegacyName || other.allocateByLegacyName) // resources are not available
+            return Objects.equals(this.legacyName, other.legacyName);
+
         if (this.vcpu < other.vcpu) return false;
         if (this.memoryGb < other.memoryGb) return false;
         if (this.diskGb < other.diskGb) return false;
@@ -124,6 +163,9 @@ public class NodeResources {
 
     /** Returns true if all the resources of this are the same as or compatible with the given resources */
     public boolean compatibleWith(NodeResources other) {
+        if (this.allocateByLegacyName || other.allocateByLegacyName) // resources are not available
+            return Objects.equals(this.legacyName, other.legacyName);
+
         if (this.vcpu != other.vcpu) return false;
         if (this.memoryGb != other.memoryGb) return false;
         if (this.diskGb != other.diskGb) return false;
@@ -137,20 +179,20 @@ public class NodeResources {
      *
      * @throws IllegalArgumentException if the given string cannot be parsed as a serial form of this
      */
-    public static NodeResources fromLegacyName(String name) {
-        if ( ! name.startsWith("d-"))
-            throw new IllegalArgumentException("A node specification string must start by 'd-' but was '" + name + "'");
-        String[] parts = name.split("-");
-        if (parts.length != 4)
-            throw new IllegalArgumentException("A node specification string must contain three numbers separated by '-' but was '" + name + "'");
-
-        double cpu = Integer.parseInt(parts[1]);
-        double mem = Integer.parseInt(parts[2]);
-        double dsk = Integer.parseInt(parts[3]);
-        if (cpu == 0) cpu = 0.5;
-        if (cpu == 2 && mem == 8 ) cpu = 1.5;
-        if (cpu == 2 && mem == 12 ) cpu = 2.3;
-        return new NodeResources(cpu, mem, dsk, DiskSpeed.fast);
+    public static NodeResources fromLegacyName(String flavorString) {
+        if (flavorString.startsWith("d-")) { // A legacy docker flavor: We still allocate by numbers
+            String[] parts = flavorString.split("-");
+            double cpu = Integer.parseInt(parts[1]);
+            double mem = Integer.parseInt(parts[2]);
+            double dsk = Integer.parseInt(parts[3]);
+            if (cpu == 0) cpu = 0.5;
+            if (cpu == 2 && mem == 8 ) cpu = 1.5;
+            if (cpu == 2 && mem == 12 ) cpu = 2.3;
+            return new NodeResources(cpu, mem, dsk, DiskSpeed.fast, false, flavorString);
+        }
+        else { // Another legacy flavor: Allocate by direct matching
+            return new NodeResources(0, 0, 0, DiskSpeed.fast, true, flavorString);
+        }
     }
 
 }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java
index 9fcee6b60ed..9e01718bfc6 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java
@@ -154,6 +154,7 @@ public class AllocatedHostsSerializer {
     }
 
     private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) {
+        if ( ! diskSpeed.valid()) return NodeResources.DiskSpeed.fast; // TODO: Remove this line after June 2019
         switch (diskSpeed.asString()) {
             case "fast" : return NodeResources.DiskSpeed.fast;
             case "slow" : return NodeResources.DiskSpeed.slow;
diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def
index 131c23054a2..1cfb18d2cd2 100644
--- a/config-provisioning/src/main/resources/configdefinitions/flavors.def
+++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def
@@ -7,14 +7,22 @@ namespace=config.provisioning
 # If a certain flavor has no config it is not necessary to list it here to use it.
 flavor[].name string
 
-# NOT USED: TODO: Remove after August 2019
+# Names of other flavors (whether mentioned in this config or not) which this flavor
+# is a replacement for: If one of these flavor names are requested, this flavor may
+# be assigned instead.
+# Replacements are transitive: If flavor a replaces b replaces c, then a request for flavor
+# c may be satisfied by assigning nodes of flavor a.
 flavor[].replaces[].name string
 
 # The monthly Total Cost of Ownership (TCO) in USD. Typically calculated as TCO divided by
 # the expected lifetime of the node (usually three years).
 flavor[].cost int default=0
 
-# NOT USED: TODO: Remove after August 2019
+# A stock flavor is any flavor which we expect to buy more of in the future.
+# Stock flavors are assigned to applications by cost priority.
+# 
+# Non-stock flavors are used for nodes for which a fixed amount has already been purchased
+# for some historical reason. These nodes are assigned to applications by exact match and ignoring cost.
 flavor[].stock bool default=true
 
 # The type of node: BARE_METAL, VIRTUAL_MACHINE or DOCKER_CONTAINER
@@ -35,6 +43,6 @@ flavor[].fastDisk bool default=true
 # Expected network interface bandwidth available for this flavor, in Mbit/s.
 flavor[].bandwidth double default=0.0
 
-# NOT USED: TODO: Remove after August 2019
+# The flavor is retired and should no longer be used.
 flavor[].retired bool default=false
 
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java
index ec3f73a8194..55ffa821e26 100644
--- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java
@@ -2,22 +2,47 @@
 package com.yahoo.config.provision;
 
 import com.yahoo.config.provisioning.FlavorsConfig;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.junit.Assert.assertEquals;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
 
 public class NodeFlavorsTest {
 
+    @Rule
+    public final ExpectedException exception = ExpectedException.none();
+
+    @Test
+    public void testReplacesWithBadValue() {
+        FlavorsConfig.Builder builder = new FlavorsConfig.Builder();
+        List<FlavorsConfig.Flavor.Builder> flavorBuilderList = new ArrayList<>();
+        FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder();
+        FlavorsConfig.Flavor.Replaces.Builder flavorReplacesBuilder = new FlavorsConfig.Flavor.Replaces.Builder();
+        flavorReplacesBuilder.name("non-existing-config");
+        flavorBuilder.name("strawberry").cost(2).replaces.add(flavorReplacesBuilder);
+        flavorBuilderList.add(flavorBuilder);
+        builder.flavor(flavorBuilderList);
+        FlavorsConfig config = new FlavorsConfig(builder);
+        exception.expect(IllegalStateException.class);
+        exception.expectMessage("Replaces for strawberry pointing to a non existing flavor: non-existing-config");
+        new NodeFlavors(config);
+    }
+
     @Test
     public void testConfigParsing() {
         FlavorsConfig.Builder builder = new FlavorsConfig.Builder();
         List<FlavorsConfig.Flavor.Builder> flavorBuilderList = new ArrayList<>();
         {
             FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder();
-            flavorBuilder.name("strawberry").cost(2);
+            FlavorsConfig.Flavor.Replaces.Builder flavorReplacesBuilder = new FlavorsConfig.Flavor.Replaces.Builder();
+            flavorReplacesBuilder.name("banana");
+            flavorBuilder.name("strawberry").cost(2).replaces.add(flavorReplacesBuilder);
             flavorBuilderList.add(flavorBuilder);
         }
         {
@@ -28,7 +53,28 @@ public class NodeFlavorsTest {
         builder.flavor(flavorBuilderList);
         FlavorsConfig config = new FlavorsConfig(builder);
         NodeFlavors nodeFlavors = new NodeFlavors(config);
-        assertEquals(3, nodeFlavors.getFlavor("banana").get().cost());
+        assertThat(nodeFlavors.getFlavor("banana").get().cost(), is(3));
+    }
+
+    @Test
+    public void testRetiredFlavorWithoutReplacement() {
+        FlavorsConfig.Builder builder = new FlavorsConfig.Builder();
+        List<FlavorsConfig.Flavor.Builder> flavorBuilderList = new ArrayList<>();
+        {
+            FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder();
+            flavorBuilder.name("retired").retired(true);
+            flavorBuilderList.add(flavorBuilder);
+        }
+        {
+            FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder();
+            flavorBuilder.name("chocolate");
+            flavorBuilderList.add(flavorBuilder);
+        }
+        builder.flavor(flavorBuilderList);
+        FlavorsConfig config = new FlavorsConfig(builder);
+        exception.expect(IllegalStateException.class);
+        exception.expectMessage("Flavor 'retired' is retired, but has no replacement");
+        new NodeFlavors(config);
     }
 
 }