diff options
author | toby <smorgrav@yahoo-inc.com> | 2017-09-21 13:14:59 +0200 |
---|---|---|
committer | toby <smorgrav@yahoo-inc.com> | 2017-10-10 13:37:59 +0200 |
commit | 68be3728309ab82c817f4308c5a3b7d1decfd8e9 (patch) | |
tree | e18a4679f87a8db8c23ffe39b9462d850c286024 /controller-api | |
parent | 84d41e7604041d8439655dcd47463db9d8c21e71 (diff) |
Refactor cost APIs and modelling - this is a breaking change
Diffstat (limited to 'controller-api')
12 files changed, 246 insertions, 188 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java index e862bd744dc..c8f913d82a8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.yahoo.vespa.hosted.controller.api.cost.CostJsonModel; +import com.yahoo.vespa.hosted.controller.api.integration.cost.restapi.CostJsonModel; import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java deleted file mode 100644 index 9bc9cfa8ed0..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.cost; - -import java.util.HashMap; -import java.util.Map; - -/** - * Cost data model for an application instance. I.e one running vespa application in one zone. - * - * @author smorgrav - */ -// TODO: Make immutable -// TODO: Make the Application own this and rename to Cost -// TODO: Enforce constraints -// TODO: Remove application id elements -// TODO: Model zone as Zone -// TODO: Cost per zone + total -// TODO: Use doubles -public class ApplicationCost { - - /** This contains environment.region */ - private String zone; - - private String tenant; - - // This must contain applicationName.instanceName. TODO: Fix - private String app; - - private int tco; - private float utilization; - private float waste; - Map<String, ClusterCost> cluster; - - /** Create an empty (invalid) application cost */ - public ApplicationCost() {} - - public ApplicationCost(String zone, String tenant, String app, int tco, float utilization, float waste, - Map<String, ClusterCost> clusterCost) { - this.zone = zone; - this.tenant = tenant; - this.app = app; - this.tco = tco; - this.utilization = utilization; - this.waste = waste; - cluster = new HashMap<>(clusterCost); - } - - public String getZone() { - return zone; - } - - public void setZone(String zone) { - this.zone = zone; - } - - public String getApp() { - return app; - } - - public void setApp(String app) { - this.app = app; - } - - public Map<String, ClusterCost> getCluster() { - return cluster; - } - - public void setCluster(Map<String, ClusterCost> cluster) { - this.cluster = cluster; - } - - public int getTco() { - return tco; - } - - public void setTco(int tco) { - if (tco < 0) throw new IllegalArgumentException("TCO cannot be negative"); - this.tco = tco; - } - - public String getTenant() { - return tenant; - } - - public void setTenant(String tenant) { - this.tenant = tenant; - } - - public float getUtilization() { - return utilization; - } - - public void setUtilization(float utilization) { - if (utilization < 0) throw new IllegalArgumentException("Utilization cannot be negative"); - this.utilization = utilization; - } - - public float getWaste() { - return waste; - } - - public void setWaste(float waste) { - this.waste = waste; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java deleted file mode 100644 index d9edf22d42c..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.cost; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException; - -import java.util.List; - -/** - * Interface for retrieving cost data directly or indirectly from yamas and - * the noderepository. - * - * - * @author smorgrav - */ -public interface Backend { - List<ApplicationCost> getApplicationCost(); - ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId appId) throws NotFoundCheckedException; -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java index 7297b60de5c..27fa3da7b3f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java @@ -4,40 +4,47 @@ package com.yahoo.vespa.hosted.controller.api.integration.cost; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException; import java.util.List; +import java.util.Map; /** - * Cost domain model declaration + * Cost domain model. + * + * Cost refers to the total cost ownership aka TCO. + * + * We define a target utilization for each cluster in an application and compares this + * to the actual utilization to get a number for ideal cost (if ideally scaled) and waste. + * + * The target utilization is defined with the following in mind: + * 1 Application stats to see contention on CPU above 80% + * 2 It is scaled for a 50/50 load balancing between two zones (thus must be able to serve the other zone) + * 3 Peaks are 2x average wrt CPU + * 4 Memory contention is rising when over 80% * * @author smorgrav */ public interface Cost { /** - * Calculate a list of the applications that is wasting most - * in absolute terms. To improve utilization, it should make - * sense to focus on this list. - * - * @return An ordered set of applications with the highest potential for - * improved CPU utilization across all environments and regions. - */ - List<ApplicationCost> getCPUAnalysis(int nofApplications); - - /** - * Collect all information and format it as a Cvs blob for download. + * Collect all information and format it as a CSV blob for download. * * @return A String with comma separated values. Can be big! */ - String getCsvForLocalAnalysis(); + default String getCsvForLocalAnalysis() { + return null; + } /** - * Get application costs for all applications across all regions and environments + * Get application costs for all applications across all zones * - * @return A list of applications in given zone + * @return A list of all application costs */ - List<ApplicationCost> getApplicationCost(); + default List<CostApplication> getApplicationCost() { + return null; + } /** * Get application costs for a given application instance in a given zone. @@ -48,6 +55,35 @@ public interface Cost { * * @return A list of applications in given zone */ - ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId app) - throws NotFoundCheckedException; + default CostApplication getApplicationCost(Environment env, RegionName region, ApplicationId app) + throws NotFoundCheckedException { + return null; + } + + /** + * Provides target utilization - default targets ARE XXX + * + * @return + */ + default CostHardwareInfo getUsageTarget(Environment env, RegionName region, ApplicationId app) { + return new CostHardwareInfo(0.8, 0.3, 0.4, 0.3); + } + + + /** + * Provides information about the clusters in the application like + * what hardware that it is using, the TCO for the hardware and number of hosts. + * + * @return Map between clusterid -> costclusterinfo + */ + Map<String, CostClusterInfo> getClusterInfo(Zone zone, ApplicationId app); + + /** + * Provides measurements as absolute numbers of hardware resources. + * + * Used to calculate the utilization of the hardware. + * + * @return Map between clusterid -> costhardwareinfo + */ + Map<String, CostHardwareInfo> getUsageMetrics(Zone zone, ApplicationId app); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostApplication.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostApplication.java new file mode 100644 index 00000000000..f41a3d7e183 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostApplication.java @@ -0,0 +1,72 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.cost; + +import com.yahoo.config.provision.Zone; + +import java.util.HashMap; +import java.util.Map; + +/** + * Cost data model for an application instance. I.e one running vespa application in one zone. + * + * @author smorgrav + */ +// TODO: Make the Application own this and rename to Cost +// TODO: Enforce constraints +// TODO: Remove application id elements +public class CostApplication { + + /** This contains environment.region */ + private final Zone zone; + + private final String tenant; + + // This must contain applicationName.instanceName. TODO: Fix + private final String app; + + private final int tco; + private final double utilization; + private final double waste; + private final Map<String, CostCluster> cluster; + + + public CostApplication(Zone zone, String tenant, String app, int tco, float utilization, float waste, + Map<String, CostCluster> clusterCost) { + if (utilization < 0) throw new IllegalArgumentException("Utilization cannot be negative"); + this.zone = zone; + this.tenant = tenant; + this.app = app; + this.tco = tco; + this.utilization = utilization; + this.waste = waste; + cluster = new HashMap<>(clusterCost); + } + + public Zone getZone() { + return zone; + } + + public String getApp() { + return app; + } + + public Map<String, CostCluster> getCluster() { + return cluster; + } + + public int getTco() { + return tco; + } + + public String getTenant() { + return tenant; + } + + public double getUtilization() { + return utilization; + } + + public double getWaste() { + return waste; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostCluster.java index 1e41325a4fd..e9920f36bca 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostCluster.java @@ -4,35 +4,34 @@ package com.yahoo.vespa.hosted.controller.api.integration.cost; import java.util.List; /** - * Cost data model for a cluster. I.e one cluster within one vespa application in one zone. + * Cost data model for a cluster. I.e one cluster within one Vespa application in one zone. * * @author smorgrav */ -// TODO: Use doubles // TODO: Make immutable // TODO: Enforce constraints // TODO: Document content -public class ClusterCost { +public class CostCluster { private int count; private String resource; - private float utilization; + private double utilization; private int tco; private String flavor; private int waste; private String type; - private float utilMem; - private float utilCpu; - private float utilDisk; - private float utilDiskBusy; - private float usageMem; - private float usageCpu; - private float usageDisk; - private float usageDiskBusy; + private double utilMem; + private double utilCpu; + private double utilDisk; + private double utilDiskBusy; + private double usageMem; + private double usageCpu; + private double usageDisk; + private double usageDiskBusy; private List<String> hostnames; /** Create an empty (invalid) cluster cost */ - public ClusterCost() {} + public CostCluster() {} public int getCount() { return count; @@ -82,7 +81,7 @@ public class ClusterCost { this.type = type; } - public float getUtilization() { + public double getUtilization() { return utilization; } @@ -99,7 +98,7 @@ public class ClusterCost { this.waste = waste; } - public float getUsageCpu() { + public double getUsageCpu() { return usageCpu; } @@ -108,7 +107,7 @@ public class ClusterCost { this.usageCpu = usageCpu; } - public float getUsageDisk() { + public double getUsageDisk() { return usageDisk; } @@ -117,7 +116,7 @@ public class ClusterCost { this.usageDisk = usageDisk; } - public float getUsageMem() { + public double getUsageMem() { return usageMem; } @@ -126,7 +125,7 @@ public class ClusterCost { this.usageMem = usageMem; } - public float getUtilCpu() { + public double getUtilCpu() { return utilCpu; } @@ -135,7 +134,7 @@ public class ClusterCost { this.utilCpu = utilCpu; } - public float getUtilDisk() { + public double getUtilDisk() { return utilDisk; } @@ -144,7 +143,7 @@ public class ClusterCost { this.utilDisk = utilDisk; } - public float getUtilMem() { + public double getUtilMem() { return utilMem; } @@ -153,7 +152,7 @@ public class ClusterCost { this.utilMem = utilMem; } - public float getUsageDiskBusy() { + public double getUsageDiskBusy() { return usageDiskBusy; } @@ -162,7 +161,7 @@ public class ClusterCost { this.usageDiskBusy = usageDiskBusy; } - public float getUtilDiskBusy() { + public double getUtilDiskBusy() { return utilDiskBusy; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostClusterInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostClusterInfo.java new file mode 100644 index 00000000000..26b703347f1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostClusterInfo.java @@ -0,0 +1,37 @@ +package com.yahoo.vespa.hosted.controller.api.integration.cost;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; + +import java.util.List; + +/** + * Value object of static cluster information. + * + * Used to calculate cost and annotate results. + * + * @author smorgrav + */ +public class CostClusterInfo { + private final Flavor flavor; + private final ClusterSpec.Type clusterType; + private final List<String> hostnames; + + CostClusterInfo(Flavor flavor, ClusterSpec.Type clusterType, List<String> hostnames) { + this.flavor = flavor; + this.clusterType = clusterType; + this.hostnames = hostnames; + } + + public Flavor getFlavor() { + return flavor; + } + + public ClusterSpec.Type getClusterType() { + return clusterType; + } + + public List<String> getHostnames() { + return hostnames; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostHardwareInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostHardwareInfo.java new file mode 100644 index 00000000000..1035c081376 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostHardwareInfo.java @@ -0,0 +1,39 @@ +package com.yahoo.vespa.hosted.controller.api.integration.cost;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * Value object for hardware resources. + * + * Can be for actual readings or target numbers. + * + * @author smorgrav + */ +public class CostHardwareInfo { + + private final double memoryGb; + private final double cpuCores; + private final double diskGb; + private final double diskBusyPercentage; + + public CostHardwareInfo(double memoryGb, double cpuCores, double diskGb, double diskBusyPercentage) { + this.memoryGb = memoryGb; + this.cpuCores = cpuCores; + this.diskGb = diskGb; + this.diskBusyPercentage = diskBusyPercentage; + } + + public double getMemoryGb() { + return memoryGb; + } + + public double getCpuCores() { + return cpuCores; + } + + public double getDiskGb() { + return diskGb; + } + + public double getDiskBusyPercentage() { + return diskBusyPercentage; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/restapi/CostJsonModel.java index bfc451946f6..d384baa94ca 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/restapi/CostJsonModel.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.cost; +package com.yahoo.vespa.hosted.controller.api.integration.cost.restapi; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -26,9 +26,9 @@ public class CostJsonModel { @JsonProperty public int tco; @JsonProperty - public float utilization; + public double utilization; @JsonProperty - public float waste; + public double waste; @JsonProperty public Map<String, Cluster> cluster; } @@ -41,7 +41,7 @@ public class CostJsonModel { @JsonProperty public String resource; @JsonProperty - public float utilization; + public double utilization; @JsonProperty public int tco; @JsonProperty @@ -62,12 +62,12 @@ public class CostJsonModel { public static class HardwareResources { @JsonProperty - public float mem; + public double mem; @JsonProperty - public float disk; + public double disk; @JsonProperty - public float cpu; + public double cpu; @JsonProperty("diskbusy") - public float diskBusy; + public double diskBusy; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/restapi/CostJsonModelAdapter.java index 088b1fa12bc..f02f8582c52 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/restapi/CostJsonModelAdapter.java @@ -1,30 +1,31 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.cost; +package com.yahoo.vespa.hosted.controller.api.integration.cost.restapi; import com.yahoo.slime.Cursor; -import com.yahoo.vespa.hosted.controller.api.cost.CostJsonModel; +import com.yahoo.vespa.hosted.controller.api.integration.cost.CostApplication; +import com.yahoo.vespa.hosted.controller.api.integration.cost.CostCluster; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; /** - * Converting from cost data model to the JSON data model used in the cost REST API. + * Serializing and deserializing cost model * * @author smorgrav */ public class CostJsonModelAdapter { - public static CostJsonModel.Application toJsonModel(ApplicationCost appCost) { + public static CostJsonModel.Application toJsonModel(CostApplication appCost) { CostJsonModel.Application app = new CostJsonModel.Application(); - app.zone = appCost.getZone(); + app.zone = appCost.getZone().toString(); app.tenant = appCost.getTenant(); app.app = appCost.getApp(); app.tco = appCost.getTco(); app.utilization = appCost.getUtilization(); app.waste = appCost.getWaste(); app.cluster = new HashMap<>(); - Map<String, ClusterCost> clusterMap = appCost.getCluster(); + Map<String, CostCluster> clusterMap = appCost.getCluster(); for (String key : clusterMap.keySet()) { app.cluster.put(key, toJsonModel(clusterMap.get(key))); } @@ -32,19 +33,19 @@ public class CostJsonModelAdapter { return app; } - public static void toSlime(ApplicationCost appCost, Cursor object) { - object.setString("zone", appCost.getZone()); + public static void toSlime(CostApplication appCost, Cursor object) { + object.setString("zone", appCost.getZone().toString()); object.setString("tenant", appCost.getTenant()); object.setString("app", appCost.getApp()); object.setLong("tco", appCost.getTco()); object.setDouble("utilization", appCost.getUtilization()); object.setDouble("waste", appCost.getWaste()); Cursor clustersObject = object.setObject("cluster"); - for (Map.Entry<String, ClusterCost> clusterEntry : appCost.getCluster().entrySet()) + for (Map.Entry<String, CostCluster> clusterEntry : appCost.getCluster().entrySet()) toSlime(clusterEntry.getValue(), clustersObject.setObject(clusterEntry.getKey())); } - public static CostJsonModel.Cluster toJsonModel(ClusterCost clusterCost) { + public static CostJsonModel.Cluster toJsonModel(CostCluster clusterCost) { CostJsonModel.Cluster cluster = new CostJsonModel.Cluster(); cluster.count = clusterCost.getCount(); cluster.resource = clusterCost.getResource(); @@ -67,7 +68,7 @@ public class CostJsonModelAdapter { return cluster; } - private static void toSlime(ClusterCost clusterCost, Cursor object) { + private static void toSlime(CostCluster clusterCost, Cursor object) { object.setLong("count", clusterCost.getCount()); object.setString("resource", clusterCost.getResource()); object.setDouble("utilization", clusterCost.getUtilization()); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/restapi/CostResource.java index 3cc6d682f4a..571460d666c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/restapi/CostResource.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.cost; +package com.yahoo.vespa.hosted.controller.api.integration.cost.restapi; import javax.ws.rs.GET; import javax.ws.rs.Path; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/restapi/package-info.java index 8e95bd4f6f1..3c3bb7e7076 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/restapi/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -package com.yahoo.vespa.hosted.controller.api.cost; +package com.yahoo.vespa.hosted.controller.api.integration.cost.restapi; import com.yahoo.osgi.annotation.ExportPackage; |