Merge pull request #11406 from vespa-engine/freva/add-remote-storage

Use resources in node response

5 files changed, 298 insertions, 127 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
index e8ae7c181d8..43fea2b76fd 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
@@ -4,8 +4,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
-import org.jetbrains.annotations.TestOnly;
 
 import java.util.Objects;
 import java.util.Optional;
@@ -22,6 +22,7 @@ public class Node {
     private final Optional<HostName> parentHostname;
     private final State state;
     private final NodeType type;
+    private final NodeResources resources;
     private final Optional<ApplicationId> owner;
     private final Version currentVersion;
     private final Version wantedVersion;
@@ -32,24 +33,20 @@ public class Node {
     private final long wantedRestartGeneration;
     private final long rebootGeneration;
     private final long wantedRebootGeneration;
-    private final double vcpu;
-    private final double memoryGb;
-    private final double diskGb;
-    private final double bandwidthGbps;
-    private final boolean fastDisk;
     private final int cost;
-    private final String canonicalFlavor;
+    private final String flavor;
     private final String clusterId;
     private final ClusterType clusterType;
 
-    public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, Optional<ApplicationId> owner,
+    public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner,
                 Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, ServiceState serviceState,
                 long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration,
-                double vcpu, double memoryGb, double diskGb, double bandwidthGbps, boolean fastDisk, int cost, String canonicalFlavor, String clusterId, ClusterType clusterType) {
+                int cost, String flavor, String clusterId, ClusterType clusterType) {
         this.hostname = hostname;
         this.parentHostname = parentHostname;
         this.state = state;
         this.type = type;
+        this.resources = resources;
         this.owner = owner;
         this.currentVersion = currentVersion;
         this.wantedVersion = wantedVersion;
@@ -60,25 +57,12 @@ public class Node {
         this.wantedRestartGeneration = wantedRestartGeneration;
         this.rebootGeneration = rebootGeneration;
         this.wantedRebootGeneration = wantedRebootGeneration;
-        this.vcpu = vcpu;
-        this.memoryGb = memoryGb;
-        this.diskGb = diskGb;
-        this.bandwidthGbps = bandwidthGbps;
-        this.fastDisk = fastDisk;
         this.cost = cost;
-        this.canonicalFlavor = canonicalFlavor;
+        this.flavor = flavor;
         this.clusterId = clusterId;
         this.clusterType = clusterType;
     }
 
-    @TestOnly
-    public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, Optional<ApplicationId> owner,
-                Version currentVersion, Version wantedVersion) {
-        this(hostname, parentHostname, state, type, owner, currentVersion, wantedVersion,
-             Version.emptyVersion, Version.emptyVersion, ServiceState.unorchestrated, 0, 0, 0, 0,
-             2, 8, 50, 1, true, 0, "d-2-8-50", "cluster", ClusterType.container);
-    }
-
     public HostName hostname() {
         return hostname;
     }
@@ -93,6 +77,10 @@ public class Node {
         return type;
     }
 
+    public NodeResources resources() {
+        return resources;
+    }
+
     public Optional<ApplicationId> owner() {
         return owner;
     }
@@ -133,32 +121,12 @@ public class Node {
         return wantedRebootGeneration;
     }
 
-    public double vcpu() {
-        return vcpu;
-    }
-
-    public double memoryGb() {
-        return memoryGb;
-    }
-
-    public double diskGb() {
-        return diskGb;
-    }
-
-    public double bandwidthGbps() {
-        return bandwidthGbps;
-    }
-
-    public boolean fastDisk() {
-        return fastDisk;
-    }
-
     public int cost() {
         return cost;
     }
 
-    public String canonicalFlavor() {
-        return canonicalFlavor;
+    public String flavor() {
+        return flavor;
     }
 
     public String clusterId() {
@@ -210,4 +178,150 @@ public class Node {
         unknown
     }
 
+    public static class Builder {
+        private HostName hostname;
+        private Optional<HostName> parentHostname = Optional.empty();
+        private State state;
+        private NodeType type;
+        private NodeResources resources;
+        private Optional<ApplicationId> owner = Optional.empty();
+        private Version currentVersion;
+        private Version wantedVersion;
+        private Version currentOsVersion;
+        private Version wantedOsVersion;
+        private ServiceState serviceState;
+        private long restartGeneration;
+        private long wantedRestartGeneration;
+        private long rebootGeneration;
+        private long wantedRebootGeneration;
+        private int cost;
+        private String flavor;
+        private String clusterId;
+        private ClusterType clusterType;
+        
+        public Builder() { }
+
+        public Builder(Node node) {
+            this.hostname = node.hostname;
+            this.parentHostname = node.parentHostname;
+            this.state = node.state;
+            this.type = node.type;
+            this.resources = node.resources;
+            this.owner = node.owner;
+            this.currentVersion = node.currentVersion;
+            this.wantedVersion = node.wantedVersion;
+            this.currentOsVersion = node.currentOsVersion;
+            this.wantedOsVersion = node.wantedOsVersion;
+            this.serviceState = node.serviceState;
+            this.restartGeneration = node.restartGeneration;
+            this.wantedRestartGeneration = node.wantedRestartGeneration;
+            this.rebootGeneration = node.rebootGeneration;
+            this.wantedRebootGeneration = node.wantedRebootGeneration;
+            this.cost = node.cost;
+            this.flavor = node.flavor;
+            this.clusterId = node.clusterId;
+            this.clusterType = node.clusterType;
+        }
+
+        public Builder hostname(HostName hostname) {
+            this.hostname = hostname;
+            return this;
+        }
+
+        public Builder parentHostname(HostName parentHostname) {
+            this.parentHostname = Optional.ofNullable(parentHostname);
+            return this;
+        }
+
+        public Builder state(State state) {
+            this.state = state;
+            return this;
+        }
+
+        public Builder type(NodeType type) {
+            this.type = type;
+            return this;
+        }
+
+        public Builder resources(NodeResources resources) {
+            this.resources = resources;
+            return this;
+        }
+
+        public Builder owner(ApplicationId owner) {
+            this.owner = Optional.ofNullable(owner);
+            return this;
+        }
+
+        public Builder currentVersion(Version currentVersion) {
+            this.currentVersion = currentVersion;
+            return this;
+        }
+
+        public Builder wantedVersion(Version wantedVersion) {
+            this.wantedVersion = wantedVersion;
+            return this;
+        }
+
+        public Builder currentOsVersion(Version currentOsVersion) {
+            this.currentOsVersion = currentOsVersion;
+            return this;
+        }
+
+        public Builder wantedOsVersion(Version wantedOsVersion) {
+            this.wantedOsVersion = wantedOsVersion;
+            return this;
+        }
+
+        public Builder serviceState(ServiceState serviceState) {
+            this.serviceState = serviceState;
+            return this;
+        }
+
+        public Builder restartGeneration(long restartGeneration) {
+            this.restartGeneration = restartGeneration;
+            return this;
+        }
+
+        public Builder wantedRestartGeneration(long wantedRestartGeneration) {
+            this.wantedRestartGeneration = wantedRestartGeneration;
+            return this;
+        }
+
+        public Builder rebootGeneration(long rebootGeneration) {
+            this.rebootGeneration = rebootGeneration;
+            return this;
+        }
+
+        public Builder wantedRebootGeneration(long wantedRebootGeneration) {
+            this.wantedRebootGeneration = wantedRebootGeneration;
+            return this;
+        }
+
+        public Builder cost(int cost) {
+            this.cost = cost;
+            return this;
+        }
+
+        public Builder flavor(String flavor) {
+            this.flavor = flavor;
+            return this;
+        }
+
+        public Builder clusterId(String clusterId) {
+            this.clusterId = clusterId;
+            return this;
+        }
+
+        public Builder clusterType(ClusterType clusterType) {
+            this.clusterType = clusterType;
+            return this;
+        }
+
+        public Node build() {
+            return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion,
+                    wantedOsVersion, serviceState, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration,
+                    cost, flavor, clusterId, clusterType);
+        }
+    }
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
index 2dcbf4ca930..94616fd27b2 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList;
@@ -77,10 +78,18 @@ public interface NodeRepository {
                                   .map(owner -> ApplicationId.from(owner.getTenant(), owner.getApplication(),
                                                                    owner.getInstance()));
         var parentHostname = Optional.ofNullable(node.getParentHostname()).map(HostName::from);
+        var resources = new NodeResources(
+                toDouble(node.getResources().getVcpu()),
+                toDouble(node.getResources().getMemoryGb()),
+                toDouble(node.getResources().getDiskGb()),
+                toDouble(node.getResources().getBandwidthGbps()),
+                diskSpeedFromString(node.getResources().getDiskSpeed()),
+                storageTypeFromString(node.getResources().getStorageType()));
         return new Node(HostName.from(node.getHostname()),
                         parentHostname,
                         fromJacksonState(node.getState()),
                         fromJacksonType(node.getType()),
+                        resources,
                         application,
                         versionFrom(node.getVespaVersion()),
                         versionFrom(node.getWantedVespaVersion()),
@@ -91,13 +100,8 @@ public interface NodeRepository {
                         toInt(node.getRestartGeneration()),
                         toInt(node.getCurrentRebootGeneration()),
                         toInt(node.getRebootGeneration()),
-                        toDouble(node.getMinCpuCores()),
-                        toDouble(node.getMinMainMemoryAvailableGb()),
-                        toDouble(node.getMinDiskAvailableGb()),
-                        toDouble(node.getBandwidthGbps()),
-                        toBoolean(node.getFastDisk()),
                         toInt(node.getCost()),
-                        node.getCanonicalFlavor(),
+                        node.getFlavor(),
                         clusterIdOf(node.getMembership()),
                         clusterTypeOf(node.getMembership()));
     }
@@ -143,6 +147,26 @@ public interface NodeRepository {
         return Node.State.unknown;
     }
 
+    private static NodeResources.DiskSpeed diskSpeedFromString(String diskSpeed) {
+        if (diskSpeed == null) return NodeResources.DiskSpeed.getDefault();
+        switch (diskSpeed) {
+            case "fast": return NodeResources.DiskSpeed.fast;
+            case "slow": return NodeResources.DiskSpeed.slow;
+            case "any": return NodeResources.DiskSpeed.any;
+            default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'");
+        }
+    }
+
+    private static NodeResources.StorageType storageTypeFromString(String storageType) {
+        if (storageType == null) return NodeResources.StorageType.getDefault();
+        switch (storageType) {
+            case "remote": return NodeResources.StorageType.remote;
+            case "local": return NodeResources.StorageType.local;
+            case "any": return NodeResources.StorageType.any;
+            default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'");
+        }
+    }
+
     private static Node.ServiceState fromBoolean(Boolean allowedDown) {
         return (allowedDown == null)
                 ? Node.ServiceState.unorchestrated
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
index 227f35498ce..78a64b98e2b 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
@@ -30,8 +30,10 @@ public class NodeRepositoryNode {
     private String openStackId;
     @JsonProperty("flavor")
     private String flavor;
-    @JsonProperty("canonicalFlavor")
-    private String canonicalFlavor;
+    @JsonProperty("resources")
+    private NodeResources resources;
+    @JsonProperty("requestedResources")
+    private NodeResources requestedResources;
     @JsonProperty("membership")
     private NodeMembership membership;
     @JsonProperty("owner")
@@ -68,20 +70,8 @@ public class NodeRepositoryNode {
     private Boolean wantToRetire;
     @JsonProperty("wantToDeprovision")
     private Boolean wantToDeprovision;
-    @JsonProperty("minDiskAvailableGb")
-    private Double minDiskAvailableGb;
-    @JsonProperty("minMainMemoryAvailableGb")
-    private Double minMainMemoryAvailableGb;
     @JsonProperty("cost")
     private Integer cost;
-    @JsonProperty("minCpuCores")
-    private Double minCpuCores;
-    @JsonProperty("bandwidthGbps")
-    private Double bandwidthGbps;
-    @JsonProperty("fastDisk")
-    private Boolean fastDisk;
-    @JsonProperty("description")
-    private String description;
     @JsonProperty("history")
     private NodeHistory[] history;
     @JsonProperty("allowedToBeDown")
@@ -155,12 +145,20 @@ public class NodeRepositoryNode {
         this.flavor = flavor;
     }
 
-    public String getCanonicalFlavor() {
-        return canonicalFlavor;
+    public NodeResources getResources() {
+        return resources;
     }
 
-    public void setCanonicalFlavor(String canonicalFlavor) {
-        this.canonicalFlavor = canonicalFlavor;
+    public void setResources(NodeResources resources) {
+        this.resources = resources;
+    }
+
+    public NodeResources getRequestedResources() {
+        return requestedResources;
+    }
+
+    public void setRequestedResources(NodeResources requestedResources) {
+        this.requestedResources = requestedResources;
     }
 
     public NodeMembership getMembership() {
@@ -289,22 +287,6 @@ public class NodeRepositoryNode {
         this.wantToDeprovision = wantToDeprovision;
     }
 
-    public Double getMinDiskAvailableGb() {
-        return minDiskAvailableGb;
-    }
-
-    public void setMinDiskAvailableGb(Double minDiskAvailableGb) {
-        this.minDiskAvailableGb = minDiskAvailableGb;
-    }
-
-    public Double getMinMainMemoryAvailableGb() {
-        return minMainMemoryAvailableGb;
-    }
-
-    public void setMinMainMemoryAvailableGb(Double minMainMemoryAvailableGb) {
-        this.minMainMemoryAvailableGb = minMainMemoryAvailableGb;
-    }
-
     public Integer getCost() {
         return cost;
     }
@@ -313,38 +295,6 @@ public class NodeRepositoryNode {
         this.cost = cost;
     }
 
-    public Double getMinCpuCores() {
-        return minCpuCores;
-    }
-
-    public void setMinCpuCores(Double minCpuCores) {
-        this.minCpuCores = minCpuCores;
-    }
-
-    public Double getBandwidthGbps() {
-        return bandwidthGbps;
-    }
-
-    public void setBandwidthGbps(Double bandwidthGbps) {
-        this.bandwidthGbps = bandwidthGbps;
-    }
-
-    public Boolean getFastDisk() {
-        return fastDisk;
-    }
-
-    public void setFastDisk(Boolean fastDisk) {
-        this.fastDisk = fastDisk;
-    }
-
-    public String getDescription() {
-        return description;
-    }
-
-    public void setDescription(String description) {
-        this.description = description;
-    }
-
     public NodeHistory[] getHistory() {
         return history;
     }
@@ -400,7 +350,8 @@ public class NodeRepositoryNode {
                ", additionalIpAddresses=" + additionalIpAddresses +
                ", openStackId='" + openStackId + '\'' +
                ", flavor='" + flavor + '\'' +
-               ", canonicalFlavor='" + canonicalFlavor + '\'' +
+               ", resources=" + resources +
+               ", requestedResources=" + requestedResources +
                ", membership=" + membership +
                ", owner=" + owner +
                ", restartGeneration=" + restartGeneration +
@@ -419,13 +370,7 @@ public class NodeRepositoryNode {
                ", parentHostname='" + parentHostname + '\'' +
                ", wantToRetire=" + wantToRetire +
                ", wantToDeprovision=" + wantToDeprovision +
-               ", minDiskAvailableGb=" + minDiskAvailableGb +
-               ", minMainMemoryAvailableGb=" + minMainMemoryAvailableGb +
                ", cost=" + cost +
-               ", minCpuCores=" + minCpuCores +
-               ", bandwidthGbps=" + bandwidthGbps +
-               ", fastDisk=" + fastDisk +
-               ", description='" + description + '\'' +
                ", history=" + Arrays.toString(history) +
                ", allowedToBeDown=" + allowedToBeDown +
                ", reports=" + reports +
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeResources.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeResources.java
new file mode 100644
index 00000000000..ded61c7a94e
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeResources.java
@@ -0,0 +1,87 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.noderepository;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author freva
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class NodeResources {
+
+    @JsonProperty
+    private Double vcpu;
+    @JsonProperty
+    private Double memoryGb;
+    @JsonProperty
+    private Double diskGb;
+    @JsonProperty
+    private Double bandwidthGbps;
+    @JsonProperty
+    private String diskSpeed;
+    @JsonProperty
+    private String storageType;
+
+    public Double getVcpu() {
+        return vcpu;
+    }
+
+    public void setVcpu(Double vcpu) {
+        this.vcpu = vcpu;
+    }
+
+    public Double getMemoryGb() {
+        return memoryGb;
+    }
+
+    public void setMemoryGb(Double memoryGb) {
+        this.memoryGb = memoryGb;
+    }
+
+    public Double getDiskGb() {
+        return diskGb;
+    }
+
+    public void setDiskGb(Double diskGb) {
+        this.diskGb = diskGb;
+    }
+
+    public Double getBandwidthGbps() {
+        return bandwidthGbps;
+    }
+
+    public void setBandwidthGbps(Double bandwidthGbps) {
+        this.bandwidthGbps = bandwidthGbps;
+    }
+
+    public String getDiskSpeed() {
+        return diskSpeed;
+    }
+
+    public void setDiskSpeed(String diskSpeed) {
+        this.diskSpeed = diskSpeed;
+    }
+
+    public String getStorageType() {
+        return storageType;
+    }
+
+    public void setStorageType(String storageType) {
+        this.storageType = storageType;
+    }
+
+    @Override
+    public String toString() {
+        return "NodeResources{" +
+                "vcpu=" + vcpu +
+                ", memoryGb=" + memoryGb +
+                ", diskGb=" + diskGb +
+                ", bandwidthGbps=" + bandwidthGbps +
+                ", diskSpeed='" + diskSpeed + '\'' +
+                ", storageType='" + storageType + '\'' +
+                '}';
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java
index 5ee6df9f034..cfe0f18260a 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.api.integration.resource;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 
@@ -37,9 +38,9 @@ public class ResourceSnapshot {
 
         return new ResourceSnapshot(
                 applicationIds.iterator().next(),
-                nodes.stream().mapToDouble(Node::vcpu).sum(),
-                nodes.stream().mapToDouble(Node::memoryGb).sum(),
-                nodes.stream().mapToDouble(Node::diskGb).sum(),
+                nodes.stream().map(Node::resources).mapToDouble(NodeResources::vcpu).sum(),
+                nodes.stream().map(Node::resources).mapToDouble(NodeResources::memoryGb).sum(),
+                nodes.stream().map(Node::resources).mapToDouble(NodeResources::diskGb).sum(),
                 timestamp,
                 zoneId
         );