Merge pull request #18058 from vespa-engine/bratseth/questdb-robustness

Don't fail nodes/v2 requests if QuestDb fails

2 files changed, 31 insertions, 6 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java
index e4b85b5317e..0fb537d8462 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java
@@ -9,7 +9,10 @@ import com.yahoo.vespa.hosted.provision.applications.ScalingEvent;
 
 import java.time.Clock;
 import java.time.Duration;
+import java.util.Optional;
 import java.util.OptionalDouble;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * A cluster with its associated metrics which allows prediction about its future behavior.
@@ -19,6 +22,8 @@ import java.util.OptionalDouble;
  */
 public class ClusterModel {
 
+    private static final Logger log = Logger.getLogger(ClusterModel.class.getName());
+
     private static final Duration CURRENT_LOAD_DURATION = Duration.ofMinutes(5);
 
     static final double idealQueryCpuLoad = 0.8;
@@ -188,4 +193,24 @@ public class ClusterModel {
         return duration;
     }
 
+    /**
+     * Create a cluster model if possible and logs a warning and returns empty otherwise.
+     * This is useful in cases where it's possible to continue without the cluser model,
+     * as QuestDb is known to temporarily fail during reading of data.
+     */
+    public static Optional<ClusterModel> create(Application application,
+                                                Cluster cluster,
+                                                ClusterSpec clusterSpec,
+                                                NodeList clusterNodes,
+                                                MetricsDb metricsDb,
+                                                Clock clock) {
+        try {
+            return Optional.of(new ClusterModel(application, cluster, clusterSpec, clusterNodes, metricsDb, clock));
+        }
+        catch (Exception e) {
+            log.log(Level.WARNING, "Failed creating a cluster model for " + application + " " + cluster, e);
+            return Optional.empty();
+        }
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
index 380003affb3..9c6efd9efe6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java
@@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb;
 
 import java.net.URI;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * Serializes application information for nodes/v2/application responses
@@ -61,8 +62,7 @@ public class ApplicationSerializer {
         NodeList nodes = applicationNodes.not().retired().cluster(cluster.id());
         if (nodes.isEmpty()) return;
         ClusterResources currentResources = nodes.toResources();
-        ClusterModel clusterModel = new ClusterModel(application, cluster, nodes.clusterSpec(), nodes, metricsDb, nodeRepository.clock());
-
+        Optional<ClusterModel> clusterModel = ClusterModel.create(application, cluster, nodes.clusterSpec(), nodes, metricsDb, nodeRepository.clock());
         Cursor clusterObject = clustersObject.setObject(cluster.id().value());
         clusterObject.setString("type", nodes.clusterSpec().type().name());
         toSlime(cluster.minResources(), clusterObject.setObject("min"));
@@ -71,12 +71,12 @@ public class ApplicationSerializer {
         if (cluster.shouldSuggestResources(currentResources))
             cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested.resources(), clusterObject.setObject("suggested")));
         cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target")));
-        clusterUtilizationToSlime(clusterModel, clusterObject.setObject("utilization"));
+        clusterModel.ifPresent(model -> clusterUtilizationToSlime(model, clusterObject.setObject("utilization")));
         scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents"));
         clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus());
-        clusterObject.setLong("scalingDuration", clusterModel.scalingDuration().toMillis());
-        clusterObject.setDouble("maxQueryGrowthRate", clusterModel.maxQueryGrowthRate());
-        clusterObject.setDouble("currentQueryFractionOfMax", clusterModel.queryFractionOfMax());
+        clusterModel.ifPresent(model -> clusterObject.setLong("scalingDuration", model.scalingDuration().toMillis()));
+        clusterModel.ifPresent(model -> clusterObject.setDouble("maxQueryGrowthRate", model.maxQueryGrowthRate()));
+        clusterModel.ifPresent(model -> clusterObject.setDouble("currentQueryFractionOfMax", model.queryFractionOfMax()));
     }
 
     private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) {