diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2021-06-01 11:54:41 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-06-01 11:54:41 +0200 |
commit | b22d7c87cabc84d7dab7f24b0bff4e32dc74fae3 (patch) | |
tree | 30ff77870bc70957428e5396d388916cb228fa6c /node-repository | |
parent | 6cd4ed85630bc25615ef4fbe406bc9a230a6a924 (diff) | |
parent | 700c1ef34d6af90740e72a9d68f3c1e0db572c0d (diff) |
Merge pull request #18058 from vespa-engine/bratseth/questdb-robustness
Don't fail nodes/v2 requests if QuestDb fails
Diffstat (limited to 'node-repository')
2 files changed, 31 insertions, 6 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index e4b85b5317e..0fb537d8462 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -9,7 +9,10 @@ import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import java.time.Clock; import java.time.Duration; +import java.util.Optional; import java.util.OptionalDouble; +import java.util.logging.Level; +import java.util.logging.Logger; /** * A cluster with its associated metrics which allows prediction about its future behavior. @@ -19,6 +22,8 @@ import java.util.OptionalDouble; */ public class ClusterModel { + private static final Logger log = Logger.getLogger(ClusterModel.class.getName()); + private static final Duration CURRENT_LOAD_DURATION = Duration.ofMinutes(5); static final double idealQueryCpuLoad = 0.8; @@ -188,4 +193,24 @@ public class ClusterModel { return duration; } + /** + * Create a cluster model if possible and logs a warning and returns empty otherwise. + * This is useful in cases where it's possible to continue without the cluser model, + * as QuestDb is known to temporarily fail during reading of data. + */ + public static Optional<ClusterModel> create(Application application, + Cluster cluster, + ClusterSpec clusterSpec, + NodeList clusterNodes, + MetricsDb metricsDb, + Clock clock) { + try { + return Optional.of(new ClusterModel(application, cluster, clusterSpec, clusterNodes, metricsDb, clock)); + } + catch (Exception e) { + log.log(Level.WARNING, "Failed creating a cluster model for " + application + " " + cluster, e); + return Optional.empty(); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 380003affb3..9c6efd9efe6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import java.net.URI; import java.util.List; +import java.util.Optional; /** * Serializes application information for nodes/v2/application responses @@ -61,8 +62,7 @@ public class ApplicationSerializer { NodeList nodes = applicationNodes.not().retired().cluster(cluster.id()); if (nodes.isEmpty()) return; ClusterResources currentResources = nodes.toResources(); - ClusterModel clusterModel = new ClusterModel(application, cluster, nodes.clusterSpec(), nodes, metricsDb, nodeRepository.clock()); - + Optional<ClusterModel> clusterModel = ClusterModel.create(application, cluster, nodes.clusterSpec(), nodes, metricsDb, nodeRepository.clock()); Cursor clusterObject = clustersObject.setObject(cluster.id().value()); clusterObject.setString("type", nodes.clusterSpec().type().name()); toSlime(cluster.minResources(), clusterObject.setObject("min")); @@ -71,12 +71,12 @@ public class ApplicationSerializer { if (cluster.shouldSuggestResources(currentResources)) cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested.resources(), clusterObject.setObject("suggested"))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); - clusterUtilizationToSlime(clusterModel, clusterObject.setObject("utilization")); + clusterModel.ifPresent(model -> clusterUtilizationToSlime(model, clusterObject.setObject("utilization"))); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus()); - clusterObject.setLong("scalingDuration", clusterModel.scalingDuration().toMillis()); - clusterObject.setDouble("maxQueryGrowthRate", clusterModel.maxQueryGrowthRate()); - clusterObject.setDouble("currentQueryFractionOfMax", clusterModel.queryFractionOfMax()); + clusterModel.ifPresent(model -> clusterObject.setLong("scalingDuration", model.scalingDuration().toMillis())); + clusterModel.ifPresent(model -> clusterObject.setDouble("maxQueryGrowthRate", model.maxQueryGrowthRate())); + clusterModel.ifPresent(model -> clusterObject.setDouble("currentQueryFractionOfMax", model.queryFractionOfMax())); } private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { |