Merge pull request #25801 from vespa-engine/bjorncs/docv1-metrics

Introduce additional docv1 metrics for most common causes of failure

4 files changed, 23 insertions, 4 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index d73385831ff..416826bcd91 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -161,6 +161,9 @@ public class VespaMetricSet {
         addMetric(metrics, ContainerMetrics.HTTPAPI_PARSE_ERROR.rate());
         addMetric(metrics, ContainerMetrics.HTTPAPI_CONDITION_NOT_MET.rate());
         addMetric(metrics, ContainerMetrics.HTTPAPI_NOT_FOUND.rate());
+        addMetric(metrics, ContainerMetrics.HTTPAPI_FAILED_UNKNOWN.rate());
+        addMetric(metrics, ContainerMetrics.HTTPAPI_FAILED_INSUFFICIENT_STORAGE.rate());
+        addMetric(metrics, ContainerMetrics.HTTPAPI_FAILED_TIMEOUT.rate());
 
         addMetric(metrics, ContainerMetrics.MEM_HEAP_TOTAL.average());
         addMetric(metrics, ContainerMetrics.MEM_HEAP_FREE.average());
diff --git a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java
index f09df685aa2..145d9256fc5 100644
--- a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java
+++ b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java
@@ -88,7 +88,10 @@ public enum ContainerMetrics {
     HTTPAPI_PARSE_ERROR("httpapi_parse_error", Unit.OPERATION, "Document operations that failed due to document parse errors"),
     HTTPAPI_CONDITION_NOT_MET("httpapi_condition_not_met", Unit.OPERATION, "Document operations not applied due to condition not met"),
     HTTPAPI_NOT_FOUND("httpapi_not_found", Unit.OPERATION, "Document operations not applied due to document not found"),
-    
+    HTTPAPI_FAILED_UNKNOWN("httpapi_failed_unknown", Unit.OPERATION, "Document operations failed by unknown cause"),
+    HTTPAPI_FAILED_TIMEOUT("httpapi_failed_timeout", Unit.OPERATION, "Document operations failed by timeout"),
+    HTTPAPI_FAILED_INSUFFICIENT_STORAGE("httpapi_failed_insufficient_storage", Unit.OPERATION, "Document operations failed by insufficient storage"),
+
     MEM_HEAP_TOTAL("mem.heap.total", Unit.BYTE, "Total available heap memory"),
     MEM_HEAP_FREE("mem.heap.free", Unit.BYTE, "Free heap memory"),
     MEM_HEAP_USED("mem.heap.used", Unit.BYTE, "Currently used heap memory"),
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
index 1e2ee3968fa..373928a3e22 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
@@ -1115,8 +1115,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
         incrementMetricNumOperations(); incrementMetricNumPuts(); sampleLatency(latency);
         switch (outcome) {
             case SUCCESS -> incrementMetricSucceeded();
+            case NOT_FOUND -> incrementMetricNotFound();
             case CONDITION_FAILED -> incrementMetricConditionNotMet();
-            default -> incrementMetricFailed();
+            case TIMEOUT -> { incrementMetricFailedTimeout(); incrementMetricFailed();}
+            case INSUFFICIENT_STORAGE -> { incrementMetricFailedInsufficientStorage(); incrementMetricFailed(); }
+            case ERROR -> { incrementMetricFailedUnknown(); incrementMetricFailed(); }
         }
     }
 
@@ -1127,7 +1130,9 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
             case SUCCESS -> incrementMetricSucceeded();
             case NOT_FOUND -> incrementMetricNotFound();
             case CONDITION_FAILED -> incrementMetricConditionNotMet();
-            default -> incrementMetricFailed();
+            case TIMEOUT -> { incrementMetricFailedTimeout(); incrementMetricFailed();}
+            case INSUFFICIENT_STORAGE -> { incrementMetricFailedInsufficientStorage(); incrementMetricFailed(); }
+            case ERROR -> { incrementMetricFailedUnknown(); incrementMetricFailed(); }
         }
     }
 
@@ -1136,7 +1141,9 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
         switch (outcome) {
             case SUCCESS,NOT_FOUND -> incrementMetricSucceeded();
             case CONDITION_FAILED -> incrementMetricConditionNotMet();
-            default -> incrementMetricFailed();
+            case TIMEOUT -> { incrementMetricFailedTimeout(); incrementMetricFailed();}
+            case INSUFFICIENT_STORAGE -> { incrementMetricFailedInsufficientStorage(); incrementMetricFailed(); }
+            case ERROR -> { incrementMetricFailedUnknown(); incrementMetricFailed(); }
         }
     }
 
@@ -1150,6 +1157,9 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
     private void incrementMetricSucceeded() { incrementMetric(MetricNames.SUCCEEDED); }
     private void incrementMetricNotFound() { incrementMetric(MetricNames.NOT_FOUND); }
     private void incrementMetricParseError() { incrementMetric(MetricNames.PARSE_ERROR); }
+    private void incrementMetricFailedUnknown() { incrementMetric(MetricNames.FAILED_UNKNOWN); }
+    private void incrementMetricFailedTimeout() { incrementMetric(MetricNames.FAILED_TIMEOUT); }
+    private void incrementMetricFailedInsufficientStorage() { incrementMetric(MetricNames.FAILED_INSUFFICIENT_STORAGE); }
     private void incrementMetric(String n) { metric.add(n, 1, null); }
     private void setMetric(String n, Number v) { metric.set(n, v, null); }
 
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java
index a5987f2398e..1bb9ab4b062 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java
@@ -23,6 +23,9 @@ public final class MetricNames {
     public static final String PARSE_ERROR = PREFIX + "parse_error";
     public static final String SUCCEEDED = PREFIX + "succeeded";
     public static final String PENDING = PREFIX + "pending";
+    public static final String FAILED_UNKNOWN = FAILED + "_unknown";
+    public static final String FAILED_TIMEOUT = FAILED + "_timeout";
+    public static final String FAILED_INSUFFICIENT_STORAGE = FAILED + "_insufficient_storage";
 
     private MetricNames() {
     }