diff options
Diffstat (limited to 'config-model/src/main')
28 files changed, 86 insertions, 1454 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 62cbf61b336..f8837a4530a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import ai.vespa.metrics.set.MetricSet; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; @@ -20,7 +21,6 @@ import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; @@ -31,7 +31,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty; +import static ai.vespa.metrics.set.MetricSet.empty; + /** * This is the admin pseudo-plugin of the Vespa model, responsible for diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java index 97a6f27b759..ea3d5e55b07 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java @@ -2,17 +2,16 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metrics.set.Metric; +import ai.vespa.metrics.set.MetricSet; import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Helper class to generate config for metrics consumers. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 58e624960f9..df6fee1deb7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metrics.set.MetricSet; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; @@ -27,7 +28,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; @@ -45,6 +45,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static ai.vespa.metrics.set.MetricSet.empty; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.addMetrics; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.generateConsumers; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.toConsumerBuilder; @@ -53,7 +54,6 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.SYSTEM; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; -import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty; /** * Container cluster for metrics proxy containers. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java deleted file mode 100644 index 685ca377e70..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import ai.vespa.metrics.ContainerMetrics; -import ai.vespa.metrics.HostedNodeAdminMetrics; -import ai.vespa.metrics.SearchNodeMetrics; -import ai.vespa.metrics.StorageMetrics; - -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Metrics used for autoscaling. - * See com.yahoo.vespa.hosted.provision.autoscale.MetricsResponse - * - * @author bratseth - */ -public class AutoscalingMetrics { - - public static final MetricSet autoscalingMetricSet = create(); - - private static MetricSet create() { - List<String> metrics = new ArrayList<>(); - - metrics.add(HostedNodeAdminMetrics.CPU_UTIL.baseName()); - metrics.add(HostedNodeAdminMetrics.GPU_UTIL.baseName()); - - // Memory util - metrics.add(HostedNodeAdminMetrics.MEM_UTIL.baseName()); // node level - default - metrics.add(HostedNodeAdminMetrics.GPU_MEM_USED.baseName()); - metrics.add(HostedNodeAdminMetrics.GPU_MEM_TOTAL.baseName()); - metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); // the basis for blocking - - // Disk util - metrics.add(HostedNodeAdminMetrics.DISK_UTIL.baseName()); // node level -default - metrics.add(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); // the basis for blocking - - metrics.add(ContainerMetrics.APPLICATION_GENERATION.last()); - metrics.add(SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); - - metrics.add(ContainerMetrics.IN_SERVICE.last()); - - // Query rate - metrics.add(ContainerMetrics.QUERIES.rate()); - metrics.add(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); - - // Write rate - metrics.add(ContainerMetrics.FEED_HTTP_REQUESTS.rate()); - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate()); - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate()); - metrics.add(StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); - - return new MetricSet("autoscaling", toMetrics(metrics)); - } - - private static Set<Metric> toMetrics(List<String> metrics) { - return metrics.stream().map(Metric::new).collect(Collectors.toCollection(() -> new LinkedHashSet<>())); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java deleted file mode 100644 index 8652acbd546..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.model.admin.monitoring; - -import ai.vespa.metrics.ContainerMetrics; -import ai.vespa.metrics.SearchNodeMetrics; -import ai.vespa.metrics.Suffix; -import ai.vespa.metrics.VespaMetrics; - -import java.util.Collections; -import java.util.EnumSet; -import java.util.LinkedHashSet; -import java.util.Set; - -import static ai.vespa.metrics.Suffix.average; -import static ai.vespa.metrics.Suffix.count; -import static ai.vespa.metrics.Suffix.max; -import static ai.vespa.metrics.Suffix.ninety_five_percentile; -import static ai.vespa.metrics.Suffix.ninety_nine_percentile; -import static ai.vespa.metrics.Suffix.sum; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; - -/** - * Metrics for the 'default' consumer, which is used by default for the generic metrics api and - * other user facing apis, e.g. 'prometheus/'. - * - * @author gjoranv - */ -public class DefaultMetrics { - - public static final String defaultMetricSetId = "default"; - - public static final MetricSet defaultMetricSet = createMetricSet(); - - private static MetricSet createMetricSet() { - return new MetricSet(defaultMetricSetId, - getAllMetrics(), - Set.of(defaultVespaMetricSet)); - } - - private static Set<Metric> getAllMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - addContentMetrics(metrics); - addContainerMetrics(metrics); - addSearchChainMetrics(metrics); - return Collections.unmodifiableSet(metrics); - } - - private static void addContainerMetrics(Set<Metric> metrics) { - addMetric(metrics, ContainerMetrics.HTTP_STATUS_1XX.rate()); - addMetric(metrics, ContainerMetrics.HTTP_STATUS_2XX.rate()); - addMetric(metrics, ContainerMetrics.HTTP_STATUS_3XX.rate()); - addMetric(metrics, ContainerMetrics.HTTP_STATUS_4XX.rate()); - addMetric(metrics, ContainerMetrics.HTTP_STATUS_5XX.rate()); - addMetric(metrics, ContainerMetrics.JDISC_GC_MS.average()); - addMetric(metrics, ContainerMetrics.MEM_HEAP_FREE.average()); - } - - private static void addSearchChainMetrics(Set<Metric> metrics) { - addMetric(metrics, ContainerMetrics.QUERIES.rate()); - addMetric(metrics, ContainerMetrics.QUERY_LATENCY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile, average)); // TODO: Remove average with Vespa 9 - addMetric(metrics, ContainerMetrics.HITS_PER_QUERY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 - addMetric(metrics, ContainerMetrics.TOTAL_HITS_PER_QUERY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 - addMetric(metrics, ContainerMetrics.DEGRADED_QUERIES.rate()); - addMetric(metrics, ContainerMetrics.FAILED_QUERIES.rate()); - addMetric(metrics, ContainerMetrics.SERVER_ACTIVE_THREADS.average()); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. - } - - private static void addContentMetrics(Set<Metric> metrics) { - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_REQUESTED_DOCUMENTS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_LATENCY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_LATENCY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 - - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DISK_USAGE.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ALLOCATED_BYTES.last()); - - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); - - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_MATCHED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_RERANKED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_SETUP_TIME, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_LATENCY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_DISK_USAGE.last()); - } - - private static void addMetric(Set<Metric> metrics, String nameWithSuffix) { - metrics.add(new Metric(nameWithSuffix)); - } - - private static void addMetric(Set<Metric> metrics, VespaMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } - - private DefaultMetrics() { } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java deleted file mode 100644 index ac03fcf56cb..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import ai.vespa.metrics.ContainerMetrics; -import ai.vespa.metrics.SearchNodeMetrics; -import com.google.common.collect.ImmutableSet; - -import java.util.Set; - -/** - * Encapsulates a minimal set of Vespa metrics to be used as default for all metrics consumers. - * - * Note: most predefined metric sets use this as a child, so changing this will require updating - * e.g. the list of Vespa metrics in the Datadog integration. - * - * @author leandroalves - */ -public class DefaultVespaMetrics { - public static final MetricSet defaultVespaMetricSet = createDefaultVespaMetricSet(); - - private static MetricSet createDefaultVespaMetricSet() { - - Set<Metric> defaultContainerMetrics = - ImmutableSet.of(new Metric(ContainerMetrics.FEED_OPERATIONS.rate()) - ); - - Set<Metric> defaultContentMetrics = - ImmutableSet.of(new Metric(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_FEEDING_BLOCKED.last()) - ); - - Set<Metric> defaultMetrics = ImmutableSet.<Metric>builder() - .addAll(defaultContainerMetrics) - .addAll(defaultContentMetrics) - .build(); - - return new MetricSet("default-vespa", defaultMetrics); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java deleted file mode 100644 index b2df37c47b2..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Helper class to model a metric. - * - * @author trygve - * @author gjoranv - */ -public class Metric { - - public final String name; - public final String outputName; - public final String description; - public final Map<String, String> dimensions; - - public Metric(String name, String outputName, String description, Map<String, String> dimensions) { - this.name = name; - this.outputName = outputName; - this.description = description; - this.dimensions = Map.copyOf(dimensions); - } - - public Metric(String name, String outputName, String description) { - this(name, outputName, description, new LinkedHashMap<>()); - } - - /** - * Creates a metric with empty description - */ - public Metric(String name, String outputName) { - this(name, outputName, ""); - } - - /** - * Creates a metric with same outputname as metricname - * - * @param name The name of the metric, same name used for output name - */ - public Metric(String name) { - this(name, name); - } - - /** - * Returns a new Metric that is a combination of this and the given metric. - * New dimensions from the given metric are added, but already existing - * dimensions will be kept unchanged. - * - * @param other The metric to add dimensions from. - * @return A new metric with dimensions from this and the other. - */ - public Metric addDimensionsFrom(Metric other) { - Map<String, String> combined = new LinkedHashMap<>(dimensions); - other.dimensions.forEach( - (k, v) -> { - if (!combined.containsKey(k)) combined.put(k, v); - }); - - return new Metric(name, outputName, description, combined); - } - - - @Override - public String toString() { - return "Metric{" + - "name='" + name + '\'' + - ", outputName='" + outputName + '\'' + - ", dimensions=" + dimensions + - '}'; - } - - - /** - * Two metrics are considered equal if they have the same name. - */ - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Metric metric = (Metric) o; - - return name.equals(metric.name); - - } - - @Override - public int hashCode() { - return name.hashCode(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java deleted file mode 100644 index fb67100b435..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -import static java.util.Collections.unmodifiableMap; - -/** - * Models a metric set containing a set of metrics and child metric sets. - * This should be immutable. - * - * @author gjoranv - */ -public class MetricSet { - - private final String id; - private final Map<String, Metric> metrics; - private final Set<MetricSet> children; - - public MetricSet(String id, Collection<Metric> metrics) { - this(id, metrics, Set.of()); - } - - public MetricSet(String id, Collection<Metric> metrics, Collection<MetricSet> children) { - this.id = Objects.requireNonNull(id, "Id cannot be null or empty."); - - this.metrics = toMapByName(metrics); - this.children = new LinkedHashSet<>(children); - } - - public static MetricSet empty() { - return new MetricSet("empty", Set.of()); - } - - public final String getId() { return id; } - - /** - * Returns all metrics in this set, including all metrics in any contained metric sets. - * - * Joins this set's metrics with its child sets into a named flat map of metrics. - * In the case of duplicate metrics, the metrics directly defined in this set - * takes precedence with respect to output name, description and dimension value - * (even if they are empty), while new dimensions from the children will be added. - * - * @return all the metrics contained in this set - */ - public final Map<String, Metric> getMetrics() { - return unmodifiableMap(flatten(metrics, children)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof MetricSet that)) return false; - - return Objects.equals(id, that.id); - - } - - @Override - public int hashCode() { - return Objects.hashCode(id); - } - - private Map<String, Metric> flatten(Map<String, Metric> metrics, Set<MetricSet> children) { - Map<String, Metric> joinedMetrics = new LinkedHashMap<>(metrics); - - for (MetricSet child : children) { - child.getMetrics().forEach( - (name, metric) -> { - if (joinedMetrics.containsKey(name)) - joinedMetrics.put(name, joinedMetrics.get(name).addDimensionsFrom(metric)); - else - joinedMetrics.put(name, metric); - }); - } - return joinedMetrics; - } - - private Map<String, Metric> toMapByName(Collection<Metric> metrics) { - Map<String, Metric> metricMap = new LinkedHashMap<>(); - metrics.forEach(metric -> metricMap.put(metric.name, metric)); - return metricMap; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index c6bd4c6e295..e18156876cd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -1,21 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import ai.vespa.metrics.set.Metric; +import ai.vespa.metrics.set.MetricSet; import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.http.ValuesFetcher; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; -import static com.yahoo.vespa.model.admin.monitoring.AutoscalingMetrics.autoscalingMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetrics.defaultMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static java.util.Collections.unmodifiableList; +import static ai.vespa.metrics.set.AutoscalingMetrics.autoscalingMetricSet; +import static ai.vespa.metrics.set.DefaultMetrics.defaultMetricSet; +import static ai.vespa.metrics.set.NetworkMetrics.networkMetricSet; +import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.VespaMetricSet.vespaMetricSet; /** * A metric consumer is a set of metrics given an id that can be requested at runtime. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java deleted file mode 100644 index 21e7d35fc7c..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import ai.vespa.metrics.HostedNodeAdminMetrics; -import com.google.common.collect.ImmutableSet; - -import java.util.Set; - -/** - * @author gjoranv - */ - -// TODO: Move to hosted repo. -public class NetworkMetrics { - - public static final MetricSet networkMetricSet = createNetworkMetricSet(); - - private static MetricSet createNetworkMetricSet() { - Set<Metric> dockerNetworkMetrics = - ImmutableSet.of(new Metric(HostedNodeAdminMetrics.NET_IN_BYTES.baseName()), - new Metric(HostedNodeAdminMetrics.NET_IN_ERROR.baseName()), - new Metric(HostedNodeAdminMetrics.NET_IN_DROPPED.baseName()), - new Metric(HostedNodeAdminMetrics.NET_OUT_BYTES.baseName()), - new Metric(HostedNodeAdminMetrics.NET_OUT_ERROR.baseName()), - new Metric(HostedNodeAdminMetrics.NET_OUT_DROPPED.baseName()), - new Metric(HostedNodeAdminMetrics.BANDWIDTH_LIMIT.baseName()) - ); - - return new MetricSet("network", dockerNetworkMetrics); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java deleted file mode 100644 index 64f2a72d1b4..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import ai.vespa.metrics.HostedNodeAdminMetrics; -import com.google.common.collect.ImmutableSet; - -import java.util.Set; - -/** - * @author gjoranv - */ -// TODO: Move to hosted repo. - -public class SystemMetrics { - - public static final MetricSet systemMetricSet = createSystemMetricSet(); - - private static MetricSet createSystemMetricSet() { - Set<Metric> dockerNodeMetrics = - ImmutableSet.of(new Metric(HostedNodeAdminMetrics.CPU_UTIL.baseName()), - new Metric(HostedNodeAdminMetrics.CPU_SYS_UTIL.baseName()), - new Metric(HostedNodeAdminMetrics.CPU_THROTTLED_TIME.baseName()), - new Metric(HostedNodeAdminMetrics.CPU_THROTTLED_CPU_TIME.baseName()), - new Metric(HostedNodeAdminMetrics.CPU_VCPUS.baseName()), - new Metric(HostedNodeAdminMetrics.DISK_LIMIT.baseName()), - new Metric(HostedNodeAdminMetrics.DISK_USED.baseName()), - new Metric(HostedNodeAdminMetrics.DISK_UTIL.baseName()), - new Metric(HostedNodeAdminMetrics.MEM_LIMIT.baseName()), - new Metric(HostedNodeAdminMetrics.MEM_USED.baseName()), - new Metric(HostedNodeAdminMetrics.MEM_UTIL.baseName()), - new Metric(HostedNodeAdminMetrics.MEM_TOTAL_USED.baseName()), - new Metric(HostedNodeAdminMetrics.MEM_TOTAL_UTIL.baseName()), - new Metric(HostedNodeAdminMetrics.GPU_UTIL.baseName()), - new Metric(HostedNodeAdminMetrics.GPU_MEM_USED.baseName()), - new Metric(HostedNodeAdminMetrics.GPU_MEM_TOTAL.baseName()) - ); - - Set<Metric> nonDockerNodeMetrics = - // Disk metrics should be based on /home, or else '/' - or simply add filesystem as dimension - ImmutableSet.of(new Metric("cpu.busy.pct", HostedNodeAdminMetrics.CPU_UTIL.baseName()), - new Metric("mem.used.pct", HostedNodeAdminMetrics.MEM_UTIL.baseName()), - new Metric("mem.active.kb", HostedNodeAdminMetrics.MEM_USED.baseName()), - new Metric("mem.total.kb", HostedNodeAdminMetrics.MEM_LIMIT.baseName()), - new Metric("used.kb", HostedNodeAdminMetrics.DISK_USED.baseName()) - ); - - Set<Metric> systemMetrics = ImmutableSet.<Metric>builder() - .addAll(dockerNodeMetrics) - .addAll(nonDockerNodeMetrics) - .build(); - - return new MetricSet("system", systemMetrics); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java deleted file mode 100644 index 8a2bae364a1..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ /dev/null @@ -1,755 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import ai.vespa.metrics.ClusterControllerMetrics; -import ai.vespa.metrics.ConfigServerMetrics; -import ai.vespa.metrics.ContainerMetrics; -import ai.vespa.metrics.DistributorMetrics; -import ai.vespa.metrics.LogdMetrics; -import ai.vespa.metrics.NodeAdminMetrics; -import ai.vespa.metrics.RoutingLayerMetrics; -import ai.vespa.metrics.SearchNodeMetrics; -import ai.vespa.metrics.SentinelMetrics; -import ai.vespa.metrics.SlobrokMetrics; -import ai.vespa.metrics.StorageMetrics; -import ai.vespa.metrics.Suffix; -import ai.vespa.metrics.VespaMetrics; - -import java.util.Collections; -import java.util.EnumSet; -import java.util.LinkedHashSet; -import java.util.Set; - -import static ai.vespa.metrics.Suffix.average; -import static ai.vespa.metrics.Suffix.count; -import static ai.vespa.metrics.Suffix.last; -import static ai.vespa.metrics.Suffix.max; -import static ai.vespa.metrics.Suffix.min; -import static ai.vespa.metrics.Suffix.ninety_five_percentile; -import static ai.vespa.metrics.Suffix.ninety_nine_percentile; -import static ai.vespa.metrics.Suffix.rate; -import static ai.vespa.metrics.Suffix.sum; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static java.util.Collections.singleton; - -/** - * Encapsulates vespa service metrics. - * - * @author gjoranv - */ -public class VespaMetricSet { - - public static final MetricSet vespaMetricSet = new MetricSet("vespa", - getVespaMetrics(), - singleton(defaultVespaMetricSet)); - - private static Set<Metric> getVespaMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - metrics.addAll(getSearchNodeMetrics()); - metrics.addAll(getStorageMetrics()); - metrics.addAll(getDistributorMetrics()); - metrics.addAll(getDocprocMetrics()); - metrics.addAll(getClusterControllerMetrics()); - metrics.addAll(getSearchChainMetrics()); - metrics.addAll(getContainerMetrics()); - metrics.addAll(getConfigServerMetrics()); - metrics.addAll(getSentinelMetrics()); - metrics.addAll(getOtherMetrics()); - - return Collections.unmodifiableSet(metrics); - } - - private static Set<Metric> getSentinelMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - addMetric(metrics, SentinelMetrics.SENTINEL_RESTARTS.count()); - addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS.last()); - addMetric(metrics, SentinelMetrics.SENTINEL_UPTIME.last()); - addMetric(metrics, SentinelMetrics.SENTINEL_RUNNING, EnumSet.of(count, last)); - - return metrics; - } - - private static Set<Metric> getOtherMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - addMetric(metrics, SlobrokMetrics.SLOBROK_HEARTBEATS_FAILED.count()); - addMetric(metrics, SlobrokMetrics.SLOBROK_MISSING_CONSENSUS.count()); - - addMetric(metrics, LogdMetrics.LOGD_PROCESSED_LINES.count()); - - // Java (JRT) TLS metrics - addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_TLS_CERTIFICATE_VERIFICATION_FAILURES.baseName()); - addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_PEER_AUTHORIZATION_FAILURES.baseName()); - addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_SERVER_TLS_CONNECIONTS_ESTABLISHED.baseName()); - addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_CLIENT_TLS_CONNECTIONS_ESTABLISHED.baseName()); - addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_SERVER_UNENCRYPTED_CONNECTIONS_ESTABLISHED.baseName()); - addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED.baseName()); - - // C++ TLS metrics - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_TLS_HANDSHAKES_FAILED.count()); - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_PEER_AUTHORIZATION_FAILURES.count()); - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_CLIENT_TLS_CONNECTIONS_ESTABLISHED.count()); - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_SERVER_TLS_CONNECTIONS_ESTABLISHED.count()); - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_CLIENT_INSECURE_CONNECTIONS_ESTABLISHED.count()); - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_SERVER_INSECURE_CONNECTIONS_ESTABLISHED.count()); - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_TLS_CONNECTIONS_BROKEN.count()); - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_FAILED_TLS_CONFIG_RELOADS.count()); - // C++ capability metrics - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_RPC_CAPABILITY_CHECKS_FAILED.count()); - addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_STATUS_CAPABILITY_CHECKS_FAILED.count()); - - // C++ Fnet metrics - addMetric(metrics, StorageMetrics.VDS_SERVER_FNET_NUM_CONNECTIONS.count()); - - // NodeAdmin certificate - addMetric(metrics, NodeAdminMetrics.ENDPOINT_CERTIFICATE_EXPIRY_SECONDS.baseName()); - addMetric(metrics, NodeAdminMetrics.NODE_CERTIFICATE_EXPIRY_SECONDS.baseName()); - - // Routing layer metrics - addMetric(metrics, RoutingLayerMetrics.WORKER_CONNECTIONS.max()); // Hosted Vespa only (routing layer) - - return metrics; - } - - private static Set<Metric> getConfigServerMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - addMetric(metrics, ConfigServerMetrics.REQUESTS.count()); - addMetric(metrics, ConfigServerMetrics.FAILED_REQUESTS.count()); - addMetric(metrics, ConfigServerMetrics.LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, ConfigServerMetrics.CACHE_CONFIG_ELEMS.last()); - addMetric(metrics, ConfigServerMetrics.CACHE_CHECKSUM_ELEMS.last()); - addMetric(metrics, ConfigServerMetrics.HOSTS.last()); - addMetric(metrics, ConfigServerMetrics.DELAYED_RESPONSES.count()); - addMetric(metrics, ConfigServerMetrics.SESSION_CHANGE_ERRORS.count()); - - addMetric(metrics, ConfigServerMetrics.ZK_Z_NODES.last()); - addMetric(metrics, ConfigServerMetrics.ZK_AVG_LATENCY.last()); - addMetric(metrics, ConfigServerMetrics.ZK_MAX_LATENCY.last()); - addMetric(metrics, ConfigServerMetrics.ZK_CONNECTIONS.last()); - addMetric(metrics, ConfigServerMetrics.ZK_OUTSTANDING_REQUESTS.last()); - - // Node repository metrics - addMetric(metrics, ConfigServerMetrics.NODES_NON_ACTIVE_FRACTION.last()); - addMetric(metrics, ConfigServerMetrics.CLUSTER_COST.last()); - addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_CPU.last()); - addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_MEMORY.last()); - addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_DISK.last()); - addMetric(metrics, ConfigServerMetrics.WANT_TO_REBOOT.max()); - addMetric(metrics, ConfigServerMetrics.WANT_TO_RESTART.max()); - addMetric(metrics, ConfigServerMetrics.RETIRED.max()); - addMetric(metrics, ConfigServerMetrics.WANT_TO_CHANGE_VESPA_VERSION.max()); - addMetric(metrics, ConfigServerMetrics.HAS_WIRE_GUARD_KEY.last()); - addMetric(metrics, ConfigServerMetrics.WANT_TO_DEPROVISION.max()); - addMetric(metrics, ConfigServerMetrics.SUSPENDED.max()); - addMetric(metrics, ConfigServerMetrics.SOME_SERVICES_DOWN.max()); - addMetric(metrics, ConfigServerMetrics.NODE_FAILER_BAD_NODE.last()); - addMetric(metrics, ConfigServerMetrics.LOCK_ATTEMPT_LOCKED_LOAD, EnumSet.of(max,average)); - - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_CPU.average()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_MEM.average()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_DISK.average()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_CPU.max()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_MEM.max()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_DISK.max()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_CPU, EnumSet.of(max,average)); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_DISK, EnumSet.of(max,average)); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_MEM, EnumSet.of(max,average)); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_SKEW.last()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_PENDING_REDEPLOYMENTS.last()); - - return metrics; - } - - private static Set<Metric> getContainerMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - addMetric(metrics, ContainerMetrics.APPLICATION_GENERATION.baseName()); - - addMetric(metrics, ContainerMetrics.HANDLED_REQUESTS.count()); - addMetric(metrics, ContainerMetrics.HANDLED_LATENCY, EnumSet.of(sum, count, max)); - - addMetric(metrics, ContainerMetrics.SERVER_NUM_OPEN_CONNECTIONS, EnumSet.of(max, last, average)); // TODO: Vespa 9: Remove last - addMetric(metrics, ContainerMetrics.SERVER_NUM_CONNECTIONS, EnumSet.of(max, last, average)); // TODO: Vespa 9: Remove last - - addMetric(metrics, ContainerMetrics.SERVER_BYTES_RECEIVED, EnumSet.of(sum, count)); - addMetric(metrics, ContainerMetrics.SERVER_BYTES_SENT, EnumSet.of(sum, count)); - - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min, max - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_REJECTED_TASKS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min, max - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_ACTIVE_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last - - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MAX_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove. - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MIN_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove. - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_RESERVED_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove. - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_BUSY_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_TOTAL_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min - - addMetric(metrics, ContainerMetrics.HTTPAPI_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, ContainerMetrics.HTTPAPI_PENDING, EnumSet.of(max, sum, count)); - addMetric(metrics, ContainerMetrics.HTTPAPI_NUM_OPERATIONS.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_NUM_UPDATES.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_NUM_REMOVES.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_NUM_PUTS.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_SUCCEEDED.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_FAILED.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_PARSE_ERROR.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_CONDITION_NOT_MET.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_NOT_FOUND.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_FAILED_UNKNOWN.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_FAILED_INSUFFICIENT_STORAGE.rate()); - addMetric(metrics, ContainerMetrics.HTTPAPI_FAILED_TIMEOUT.rate()); - - addMetric(metrics, ContainerMetrics.MEM_HEAP_TOTAL.average()); - addMetric(metrics, ContainerMetrics.MEM_HEAP_FREE.average()); - addMetric(metrics, ContainerMetrics.MEM_HEAP_USED, EnumSet.of(average, max)); - addMetric(metrics, ContainerMetrics.MEM_DIRECT_TOTAL.average()); - addMetric(metrics, ContainerMetrics.MEM_DIRECT_FREE.average()); - addMetric(metrics, ContainerMetrics.MEM_DIRECT_USED, EnumSet.of(average, max)); - addMetric(metrics, ContainerMetrics.MEM_DIRECT_COUNT.max()); - addMetric(metrics, ContainerMetrics.MEM_NATIVE_TOTAL.average()); - addMetric(metrics, ContainerMetrics.MEM_NATIVE_FREE.average()); - addMetric(metrics, ContainerMetrics.MEM_NATIVE_USED.average()); - - addMetric(metrics, ContainerMetrics.JDISC_MEMORY_MAPPINGS.max()); - addMetric(metrics, ContainerMetrics.JDISC_OPEN_FILE_DESCRIPTORS.max()); - - addMetric(metrics, ContainerMetrics.JDISC_GC_COUNT, EnumSet.of(average, max, last)); // TODO: Vespa 9: Remove last - addMetric(metrics, ContainerMetrics.JDISC_GC_MS, EnumSet.of(average, max, last)); // TODO: Vespa 9: Remove last - - addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS.last()); - addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS_WITH_RETAINED_REFS.last()); - - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_IS_ACTIVE.last()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_COUNT.last()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_FAILURE_COUNT.last()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_MILLIS.last()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_DEACTIVATION_COUNT.last()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_DEACTIVATION_FAILURE_COUNT.last()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_DEACTIVATION_MILLIS.last()); - - addMetric(metrics, ContainerMetrics.ATHENZ_TENANT_CERT_EXPIRY_SECONDS.last()); - addMetric(metrics, ContainerMetrics.CONTAINER_IAM_ROLE_EXPIRY_SECONDS.baseName()); - - addMetric(metrics, ContainerMetrics.HTTP_STATUS_1XX.rate()); - addMetric(metrics, ContainerMetrics.HTTP_STATUS_2XX.rate()); - addMetric(metrics, ContainerMetrics.HTTP_STATUS_3XX.rate()); - addMetric(metrics, ContainerMetrics.HTTP_STATUS_4XX.rate()); - addMetric(metrics, ContainerMetrics.HTTP_STATUS_5XX.rate()); - - addMetric(metrics, ContainerMetrics.JDISC_HTTP_REQUEST_PREMATURELY_CLOSED.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_REQUEST_REQUESTS_PER_CONNECTION, EnumSet.of(sum, count, min, max, average)); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_REQUEST_URI_LENGTH, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_REQUEST_CONTENT_SIZE, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_REQUESTS, EnumSet.of(rate, count)); - - addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CHIFERS.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_CONNECTION_CLOSED.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_UNKNOWN.rate()); - - addMetric(metrics, ContainerMetrics.JDISC_HTTP_FILTER_RULE_BLOCKED_REQUESTS.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_FILTER_RULE_ALLOWED_REQUESTS.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_FILTERING_REQUEST_HANDLED.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_FILTERING_REQUEST_UNHANDLED.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_FILTERING_RESPONSE_HANDLED.rate()); - addMetric(metrics, ContainerMetrics.JDISC_HTTP_FILTERING_RESPONSE_UNHANDLED.rate()); - - addMetric(metrics, ContainerMetrics.JDISC_HTTP_HANDLER_UNHANDLED_EXCEPTIONS.rate()); - - addMetric(metrics, ContainerMetrics.JDISC_APPLICATION_FAILED_COMPONENT_GRAPHS.rate()); - addMetric(metrics, ContainerMetrics.JDISC_APPLICATION_COMPONENT_GRAPH_CREATION_TIME_MILLIS.last()); - addMetric(metrics, ContainerMetrics.JDISC_APPLICATION_COMPONENT_GRAPH_RECONFIGURATIONS.rate()); - - addMetric(metrics, ContainerMetrics.JDISC_JVM.last()); - - // Deprecated metrics. TODO: Remove on Vespa 9. - addMetric(metrics, ContainerMetrics.SERVER_REJECTED_REQUESTS, EnumSet.of(rate, count)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. - addMetric(metrics, ContainerMetrics.SERVER_THREAD_POOL_SIZE, EnumSet.of(max, last)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. - addMetric(metrics, ContainerMetrics.SERVER_ACTIVE_THREADS, EnumSet.of(min, max, sum, count, last)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. - - addMetric(metrics, ContainerMetrics.JDISC_TLS_CAPABILITY_CHECKS_SUCCEEDED.rate()); - addMetric(metrics, ContainerMetrics.JDISC_TLS_CAPABILITY_CHECKS_FAILED.rate()); - - return metrics; - } - - private static Set<Metric> getClusterControllerMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - addMetric(metrics, ClusterControllerMetrics.DOWN_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.INITIALIZING_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.MAINTENANCE_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.RETIRED_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.STOPPING_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.UP_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.CLUSTER_STATE_CHANGE_COUNT.baseName()); - addMetric(metrics, ClusterControllerMetrics.BUSY_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); // TODO: Vespa 9: Remove last - addMetric(metrics, ClusterControllerMetrics.IDLE_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); // TODO: Vespa 9: Remove last - - addMetric(metrics, ClusterControllerMetrics.WORK_MS, EnumSet.of(last, sum, count)); // TODO: Vespa 9: Remove last - - addMetric(metrics, ClusterControllerMetrics.IS_MASTER.last()); - addMetric(metrics, ClusterControllerMetrics.REMOTE_TASK_QUEUE_SIZE.last()); - // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. - // DO NOT RELY ON THIS METRIC YET. - addMetric(metrics, ClusterControllerMetrics.NODE_EVENT_COUNT.baseName()); - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MEMORY_LIMIT.last()); - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_DISK_LIMIT.last()); - addMetric(metrics, ClusterControllerMetrics.REINDEXING_PROGRESS.last()); - - return metrics; - } - - private static Set<Metric> getDocprocMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - // per chain - metrics.add(new Metric("documents_processed.rate")); - - return metrics; - } - - private static Set<Metric> getSearchChainMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - addMetric(metrics, ContainerMetrics.PEAK_QPS.max()); - addMetric(metrics, ContainerMetrics.SEARCH_CONNECTIONS, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.FEED_LATENCY, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.FEED_HTTP_REQUESTS, EnumSet.of(count, rate)); - addMetric(metrics, ContainerMetrics.QUERIES.rate()); - addMetric(metrics, ContainerMetrics.QUERY_CONTAINER_LATENCY, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.QUERY_LATENCY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile)); - addMetric(metrics, ContainerMetrics.QUERY_TIMEOUT, EnumSet.of(sum, count, max, min, ninety_five_percentile, ninety_nine_percentile)); - addMetric(metrics, ContainerMetrics.FAILED_QUERIES.rate()); - addMetric(metrics, ContainerMetrics.DEGRADED_QUERIES.rate()); - addMetric(metrics, ContainerMetrics.HITS_PER_QUERY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile)); - addMetric(metrics, ContainerMetrics.SEARCH_CONNECTIONS, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.QUERY_HIT_OFFSET, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.DOCUMENTS_COVERED.count()); - addMetric(metrics, ContainerMetrics.DOCUMENTS_TOTAL.count()); - addMetric(metrics, ContainerMetrics.DOCUMENTS_TARGET_TOTAL.count()); - addMetric(metrics, ContainerMetrics.JDISC_RENDER_LATENCY, EnumSet.of(min, max, count, sum, last, average)); // TODO: Vespa 9: Remove last, average - addMetric(metrics, ContainerMetrics.QUERY_ITEM_COUNT, EnumSet.of(max, sum, count)); - addMetric(metrics, ContainerMetrics.TOTAL_HITS_PER_QUERY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile)); - addMetric(metrics, ContainerMetrics.EMPTY_RESULTS.rate()); - addMetric(metrics, ContainerMetrics.REQUESTS_OVER_QUOTA, EnumSet.of(rate, count)); - addMetric(metrics, ContainerMetrics.DOCPROC_PROC_TIME, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.DOCPROC_DOCUMENTS, EnumSet.of(sum, count, max, min)); - - addMetric(metrics, ContainerMetrics.RELEVANCE_AT_1, EnumSet.of(sum, count)); - addMetric(metrics, ContainerMetrics.RELEVANCE_AT_3, EnumSet.of(sum, count)); - addMetric(metrics, ContainerMetrics.RELEVANCE_AT_10, EnumSet.of(sum, count)); - - // Errors from search container - addMetric(metrics, ContainerMetrics.ERROR_TIMEOUT.rate()); - addMetric(metrics, ContainerMetrics.ERROR_BACKENDS_OOS.rate()); - addMetric(metrics, ContainerMetrics.ERROR_PLUGIN_FAILURE.rate()); - addMetric(metrics, ContainerMetrics.ERROR_BACKEND_COMMUNICATION_ERROR.rate()); - addMetric(metrics, ContainerMetrics.ERROR_EMPTY_DOCUMENT_SUMMARIES.rate()); - addMetric(metrics, ContainerMetrics.ERROR_INVALID_QUERY_PARAMETER.rate()); - addMetric(metrics, ContainerMetrics.ERROR_INTERNAL_SERVER_ERROR.rate()); - addMetric(metrics, ContainerMetrics.ERROR_MISCONFIGURED_SERVER.rate()); - addMetric(metrics, ContainerMetrics.ERROR_INVALID_QUERY_TRANSFORMATION.rate()); - addMetric(metrics, ContainerMetrics.ERROR_RESULTS_WITH_ERRORS.rate()); - addMetric(metrics, ContainerMetrics.ERROR_UNSPECIFIED.rate()); - addMetric(metrics, ContainerMetrics.ERROR_UNHANDLED_EXCEPTION.rate()); - - return metrics; - } - - private static Set<Metric> getSearchNodeMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); - - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_REMOVED.last()); - - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_DOCS_IN_MEMORY.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DISK_USAGE.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ALLOCATED_BYTES.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_HEART_BEAT_AGE.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCSUM_DOCS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCSUM_LATENCY, EnumSet.of(max, sum, count)); - - // Search protocol - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_REQUEST_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_REPLY_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_LATENCY, EnumSet.of(max, sum, average)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_REQUEST_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_REPLY_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_REQUESTED_DOCUMENTS.count()); - - // Executors shared between all document dbs - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_PROTON_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_PROTON_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_PROTON_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_PROTON_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FLUSH_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FLUSH_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FLUSH_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FLUSH_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_MATCH_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_MATCH_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_MATCH_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_MATCH_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_DOCSUM_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_DOCSUM_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_DOCSUM_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_DOCSUM_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_SHARED_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_SHARED_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_SHARED_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_SHARED_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_WARMUP_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_WARMUP_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_WARMUP_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_WARMUP_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FIELD_WRITER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FIELD_WRITER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FIELD_WRITER_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FIELD_WRITER_UTILIZATION, EnumSet.of(max, sum, count)); - - // jobs - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_TOTAL.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_ATTRIBUTE_FLUSH.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_MEMORY_INDEX_FLUSH.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_DISK_INDEX_FUSION.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_DOCUMENT_STORE_FLUSH.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_DOCUMENT_STORE_COMPACT.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_BUCKET_MOVE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_LID_SPACE_COMPACT.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_JOB_REMOVED_DOCUMENTS_PRUNE.average()); - - // Threading service (per document db) - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_UTILIZATION, EnumSet.of(max, sum, count)); - - // lid space - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_LIMIT.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_HIGHEST_USED_LID.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_USED_LIDS.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_BLOAT_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_LIMIT.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_HIGHEST_USED_LID.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_USED_LIDS.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_BLOAT_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_LIMIT.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_HIGHEST_USED_LID.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_USED_LIDS.last()); - - // bucket move - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_BUCKET_MOVE_BUCKETS_PENDING.last()); - - // resource usage - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK_USAGE_TOTAL.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK_USAGE_TOTAL_UTILIZATION.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK_USAGE_TRANSIENT.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY_USAGE_TOTAL.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY_USAGE_TOTAL_UTILIZATION.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY_USAGE_TRANSIENT.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY_MAPPINGS.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_OPEN_FILE_DESCRIPTORS.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_FEEDING_BLOCKED.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MALLOC_ARENA.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_RESOURCE_USAGE_ADDRESS_SPACE.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_RESOURCE_USAGE_FEEDING_BLOCKED.max()); - - // CPU util - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_CPU_UTIL_SETUP, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_CPU_UTIL_READ, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_CPU_UTIL_WRITE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_CPU_UTIL_COMPACT, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_CPU_UTIL_OTHER, EnumSet.of(max, sum, count)); - - // transaction log - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_ENTRIES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_DISK_USAGE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_REPLAY_TIME.last()); - - // document store - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_DISK_USAGE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_DISK_BLOAT.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_MAX_BUCKET_SPREAD.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_MEMORY_USAGE_ONHOLD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_DISK_USAGE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_DISK_BLOAT.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MAX_BUCKET_SPREAD.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MEMORY_USAGE_ONHOLD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_DISK_USAGE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_DISK_BLOAT.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MAX_BUCKET_SPREAD.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_ONHOLD_BYTES.average()); - - // document store cache - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_MEMORY_USAGE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_HIT_RATE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_LOOKUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_INVALIDATIONS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_MEMORY_USAGE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_HIT_RATE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_LOOKUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_CACHE_INVALIDATIONS.rate()); - - // attribute - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_ONHOLD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_ATTRIBUTE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_ATTRIBUTE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_ATTRIBUTE_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_ATTRIBUTE_MEMORY_USAGE_ONHOLD_BYTES.average()); - - // index - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_ONHOLD_BYTES.average()); - - // matching - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_SOFT_DOOMED_QUERIES.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERY_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERY_SETUP_TIME, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_MATCHED, EnumSet.of(rate, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERIES.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_SOFT_DOOMED_QUERIES.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_SOFT_DOOM_FACTOR, EnumSet.of(min, max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_SETUP_TIME, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_GROUPING_TIME, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_DOCS_MATCHED, EnumSet.of(rate, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_LIMITED_QUERIES.rate()); - - // feeding - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_FEEDING_COMMIT_OPERATIONS, EnumSet.of(max, sum, count, rate)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_FEEDING_COMMIT_LATENCY, EnumSet.of(max, sum, count)); - - return metrics; - } - - private static Set<Metric> getStorageMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - - // TODO - Vespa 9: For the purpose of this file and likely elsewhere, all but the last aggregate specifier, - // TODO - Vespa 9: such as 'average' and 'sum' in the metric names below are just confusing and can be mentally - // TODO - Vespa 9: disregarded when considering metric names. Clean up for Vespa 9. - addMetric(metrics, StorageMetrics.VDS_DATASTORED_ALLDISKS_BUCKETS.average()); - addMetric(metrics, StorageMetrics.VDS_DATASTORED_ALLDISKS_DOCS.average()); - addMetric(metrics, StorageMetrics.VDS_DATASTORED_ALLDISKS_BYTES.average()); - addMetric(metrics, StorageMetrics.VDS_VISITOR_ALLTHREADS_AVERAGEVISITORLIFETIME, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_VISITOR_ALLTHREADS_AVERAGEQUEUEWAIT, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_VISITOR_ALLTHREADS_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_VISITOR_ALLTHREADS_COMPLETED.rate()); - addMetric(metrics, StorageMetrics.VDS_VISITOR_ALLTHREADS_CREATED.rate()); - addMetric(metrics, StorageMetrics.VDS_VISITOR_ALLTHREADS_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_VISITOR_ALLTHREADS_AVERAGEMESSAGESENDTIME, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_VISITOR_ALLTHREADS_AVERAGEPROCESSINGTIME, EnumSet.of(max, sum, count)); - - addMetric(metrics, StorageMetrics.VDS_FILESTOR_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_AVERAGEQUEUEWAIT, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ACTIVE_OPERATIONS_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ACTIVE_OPERATIONS_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_THROTTLE_WINDOW_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_THROTTLE_WAITING_THREADS, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_THROTTLE_ACTIVE_TOKENS, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_MERGEMETADATAREADLATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_MERGEDATAREADLATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_MERGEDATAWRITELATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_MERGE_PUT_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_MERGE_REMOVE_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLSTRIPES_THROTTLED_RPC_DIRECT_DISPATCHES.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLSTRIPES_THROTTLED_PERSISTENCE_THREAD_POLLS.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLSTRIPES_TIMEOUTS_WAITING_FOR_THROTTLE_TOKEN.rate()); - - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_TEST_AND_SET_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_REQUEST_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_TEST_AND_SET_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_REQUEST_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_GET_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_GET_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_GET_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_GET_REQUEST_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_TEST_AND_SET_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_REQUEST_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_CREATEITERATOR_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_CREATEITERATOR_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_VISIT_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_VISIT_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_LOCATION_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_LOCATION_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_SPLITBUCKETS_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_JOINBUCKETS_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_DELETEBUCKETS_COUNT.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_DELETEBUCKETS_FAILED.rate()); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_DELETEBUCKETS_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_SETBUCKETSTATES_COUNT.rate()); - - addMetric(metrics, StorageMetrics.VDS_MERGETHROTTLER_AVERAGEQUEUEWAITINGTIME, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_MERGETHROTTLER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_MERGETHROTTLER_ACTIVE_WINDOW_SIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, StorageMetrics.VDS_MERGETHROTTLER_BOUNCED_DUE_TO_BACK_PRESSURE.rate()); - addMetric(metrics, StorageMetrics.VDS_MERGETHROTTLER_LOCALLYEXECUTEDMERGES_OK.rate()); - addMetric(metrics, StorageMetrics.VDS_MERGETHROTTLER_MERGECHAINS_OK.rate()); - addMetric(metrics, StorageMetrics.VDS_MERGETHROTTLER_MERGECHAINS_FAILURES_BUSY.rate()); - addMetric(metrics, StorageMetrics.VDS_MERGETHROTTLER_MERGECHAINS_FAILURES_TOTAL.rate()); - - return metrics; - } - - private static Set<Metric> getDistributorMetrics() { - Set<Metric> metrics = new LinkedHashSet<>(); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS_RECHECKING.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_IDEALSTATE_DIFF.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS_TOOFEWCOPIES.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS_TOOMANYCOPIES.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS_NOTRUSTED.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKET_REPLICAS_MOVING_OUT.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKET_REPLICAS_COPYING_OUT.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKET_REPLICAS_COPYING_IN.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKET_REPLICAS_SYNCING.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MAX_OBSERVED_TIME_SINCE_LAST_GC_SEC.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_DELETE_BUCKET_DONE_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_DELETE_BUCKET_DONE_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_DELETE_BUCKET_PENDING.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_DONE_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_DONE_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_PENDING.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_BLOCKED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_THROTTLED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_SOURCE_ONLY_COPY_CHANGED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_SOURCE_ONLY_COPY_DELETE_BLOCKED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_SOURCE_ONLY_COPY_DELETE_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_SPLIT_BUCKET_DONE_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_SPLIT_BUCKET_DONE_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_SPLIT_BUCKET_PENDING.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_JOIN_BUCKET_DONE_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_JOIN_BUCKET_DONE_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_JOIN_BUCKET_PENDING.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_GARBAGE_COLLECTION_DONE_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_GARBAGE_COLLECTION_DONE_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_GARBAGE_COLLECTION_PENDING.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_GARBAGE_COLLECTION_DOCUMENTS_REMOVED, EnumSet.of(count, rate)); - - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_TOTAL.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_NOTFOUND.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_TEST_AND_SET_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_CONCURRENT_MUTATIONS.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_NOTCONNECTED.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_NOTREADY.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_WRONGDISTRIBUTOR.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_SAFE_TIME_NOT_REACHED.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_STORAGEFAILURE.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_TIMEOUT.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_BUSY.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_PUTS_FAILURES_INCONSISTENT_BUCKET.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_REMOVES_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_REMOVES_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_REMOVES_FAILURES_TOTAL.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_REMOVES_FAILURES_NOTFOUND.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_REMOVES_FAILURES_TEST_AND_SET_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_REMOVES_FAILURES_CONCURRENT_MUTATIONS.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_UPDATES_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_UPDATES_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_UPDATES_FAILURES_TOTAL.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_UPDATES_FAILURES_NOTFOUND.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_UPDATES_FAILURES_TEST_AND_SET_FAILED.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_UPDATES_FAILURES_CONCURRENT_MUTATIONS.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_UPDATES_DIVERGING_TIMESTAMP_UPDATES.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_REMOVELOCATIONS_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_REMOVELOCATIONS_FAILURES_TOTAL.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_GETS_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_GETS_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_GETS_FAILURES_TOTAL.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_GETS_FAILURES_NOTFOUND.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_LATENCY, EnumSet.of(max, sum, count)); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_OK.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_TOTAL.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_NOTREADY.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_NOTCONNECTED.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_WRONGDISTRIBUTOR.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_SAFE_TIME_NOT_REACHED.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_STORAGEFAILURE.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_TIMEOUT.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_BUSY.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_INCONSISTENT_BUCKET.rate()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_VISITOR_FAILURES_NOTFOUND.rate()); - - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_DOCSSTORED.average()); - addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_BYTESSTORED.average()); - - addMetric(metrics, DistributorMetrics.VDS_BOUNCER_CLOCK_SKEW_ABORTS.count()); - - return metrics; - } - - private static void addMetric(Set<Metric> metrics, String nameWithSuffix) { - metrics.add(new Metric(nameWithSuffix)); - } - - private static void addMetric(Set<Metric> metrics, VespaMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } - - private static void addMetric(Set<Metric> metrics, String metricName, Iterable<String> aggregateSuffices) { - for (String suffix : aggregateSuffices) { - metrics.add(new Metric(metricName + "." + suffix)); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java index 597a2da0f2c..3af2b4f8732 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java @@ -1,18 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring.builder; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; +import ai.vespa.metrics.set.MetricSet; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; -import static com.yahoo.vespa.model.admin.monitoring.AutoscalingMetrics.autoscalingMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetrics.defaultMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; +import static ai.vespa.metrics.set.AutoscalingMetrics.autoscalingMetricSet; +import static ai.vespa.metrics.set.DefaultMetrics.defaultMetricSet; +import static ai.vespa.metrics.set.DefaultVespaMetrics.defaultVespaMetricSet; +import static ai.vespa.metrics.set.InfrastructureMetricSet.infrastructureMetricSet; +import static ai.vespa.metrics.set.NetworkMetrics.networkMetricSet; +import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.VespaMetricSet.vespaMetricSet; /** * A data object for predefined metric sets. @@ -27,7 +28,8 @@ public class PredefinedMetricSets { vespaMetricSet, systemMetricSet, networkMetricSet, - autoscalingMetricSet + autoscalingMetricSet, + infrastructureMetricSet ); public static Map<String, MetricSet> get() { return sets; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index f75b2a864f9..a00fe47eb21 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring.builder.xml; +import ai.vespa.metrics.set.Metric; +import ai.vespa.metrics.set.MetricSet; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.text.XML; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import org.w3c.dom.Element; @@ -14,8 +14,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.DefaultVespaMetrics.defaultVespaMetricSet; +import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; + /** * @author gjoranv diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java index 0193eacba3a..d919a35c7ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java @@ -29,9 +29,6 @@ public class DeploymentSpecValidator extends Validator { DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader); List<ContainerModel> containers = model.getRoot().configModelRepo().getModels(ContainerModel.class); for (DeploymentInstanceSpec instance : deploymentSpec.instances()) { - instance.globalServiceId().ifPresent(globalServiceId -> { - requireClusterId(containers, instance.name(), "Attribute 'globalServiceId'", globalServiceId); - }); instance.endpoints().forEach(endpoint -> { requireClusterId(containers, instance.name(), "Endpoint '" + endpoint.endpointId() + "'", endpoint.containerId()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index a4a4210f8cc..cebe08288f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -56,7 +56,8 @@ public class HostedSslConnectorFactory extends ConnectorFactory { new ConnectorConfig.TlsClientAuthEnforcer.Builder() .pathWhitelist(List.of("/status.html")).enable(true)); } - connectorBuilder.ssl.enabledProtocols(TlsContext.ALLOWED_PROTOCOLS); + // Disables TLSv1.3 as it causes some browsers to prompt user for client certificate (when connector has 'want' auth) + connectorBuilder.ssl.enabledProtocols(List.of("TLSv1.2")); if (!tlsCiphersOverride.isEmpty()) { connectorBuilder.ssl.enabledCipherSuites(tlsCiphersOverride.stream().sorted().toList()); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 1036a615bb5..31f8eba48bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -10,7 +10,6 @@ import com.yahoo.config.application.Xml; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ApplicationClusterEndpoint; @@ -360,24 +359,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { context.getDeployState().getProperties().athenzDnsSuffix(), context.getDeployState().zone(), deploymentSpec); - addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec); + addRotationProperties(cluster, context.getDeployState().getEndpoints()); } - private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<ContainerEndpoint> endpoints, DeploymentSpec spec) { + private void addRotationProperties(ApplicationContainerCluster cluster, Set<ContainerEndpoint> endpoints) { cluster.getContainers().forEach(container -> { setRotations(container, endpoints, cluster.getName()); - container.setProp("activeRotation", Boolean.toString(zoneHasActiveRotation(zone, spec))); + container.setProp("activeRotation", "true"); // TODO(mpolden): This is unused and should be removed }); } - private boolean zoneHasActiveRotation(Zone zone, DeploymentSpec spec) { - Optional<DeploymentInstanceSpec> instance = spec.instance(app.getApplicationId().instance()); - if (instance.isEmpty()) return false; - return instance.get().zones().stream() - .anyMatch(declaredZone -> declaredZone.concerns(zone.environment(), Optional.of(zone.region())) && - declaredZone.active()); - } - private void setRotations(Container container, Set<ContainerEndpoint> endpoints, String containerClusterName) { var rotationsProperty = endpoints.stream() .filter(endpoint -> endpoint.clusterId().equals(containerClusterName)) @@ -594,7 +585,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addDefaultConnectorHostedFilterBinding(ApplicationContainerCluster cluster) { cluster.getHttp().getAccessControl() - .ifPresent(accessControl -> accessControl.configureDefaultHostedConnector(cluster.getHttp())); ; + .ifPresent(accessControl -> accessControl.configureDefaultHostedConnector(cluster.getHttp())); } private void addCloudMtlsConnector(DeployState state, ApplicationContainerCluster cluster) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index f3e02adff6b..be3ca0b8aa9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -42,13 +42,13 @@ public class ModelIdResolver { models.put("multilingual-e5-small-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small/tokenizer.json"); models.put("multilingual-e5-small-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/model.onnx"); - models.put("multilingual-e5-small-vocab-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/tokenizer.json"); + models.put("multilingual-e5-small-cpu-friendly-vocab", "https://data.vespa.oath.cloud/onnx_models/multilingual-e5-small-cpu-friendly/tokenizer.json"); models.put("e5-small-v2", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2/model.onnx"); models.put("e5-small-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2/tokenizer.json"); models.put("e5-small-v2-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/model.onnx"); - models.put("e5-small-v2-vocab-cpu-friendly", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/tokenizer.json"); + models.put("e5-small-v2-cpu-friendly-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-small-v2-cpu-friendly/tokenizer.json"); models.put("e5-base-v2", "https://data.vespa.oath.cloud/onnx_models/e5-base-v2/model.onnx"); models.put("e5-base-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-base-v2/tokenizer.json"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 34ea41384bc..8dc58a33e04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -35,7 +35,6 @@ import java.util.Optional; import java.util.Objects; import java.util.TreeMap; import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Encapsulates the various options for search in a content model. @@ -44,7 +43,8 @@ import java.util.stream.Collectors; public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> implements ProtonConfig.Producer, DispatchNodesConfig.Producer, - DispatchConfig.Producer + DispatchConfig.Producer, + Redundancy.Provider { private static final int DEFAULT_DOC_STORE_COMPRESSION_LEVEL = 3; @@ -303,7 +303,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, this, deployState.featureFlags()); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); @@ -312,7 +312,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> tls.initService(deployState); } else { searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, - tuning, resourceLimits, fractionOfMemoryReserved) + tuning, resourceLimits, fractionOfMemoryReserved, this) .build(deployState, parent, element.getXml()); tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } @@ -352,7 +352,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (hasIndexedCluster()) { // Important: these must all be the normalized "within a single leaf group" values, // _not_ the cluster-wide, cross-group values. - indexedCluster.setSearchableCopies(redundancy.readyCopies()); indexedCluster.setRedundancy(redundancy.finalRedundancy()); } this.redundancy = redundancy; @@ -495,4 +494,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public String toString() { return "content cluster '" + clusterName + "'"; } + public Redundancy redundancy() { return redundancy; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java index 616dcbbc760..e883e87d36c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java @@ -64,4 +64,8 @@ public class Redundancy implements StorDistributionConfig.Producer, ProtonConfig builder.distribution(distBuilder); } + public interface Provider { + Redundancy redundancy(); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 6078215f9b6..74c1b7400c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterMembership; @@ -132,8 +131,7 @@ public class StorageGroup { public void getConfig(StorDistributionConfig.Group.Builder builder) { builder.index(index == null ? "invalid" : index); builder.name(name == null ? "invalid" : name); - if (partitions.isPresent()) - builder.partitions(partitions.get()); + partitions.ifPresent(builder::partitions); for (StorageNode node : nodes) { StorDistributionConfig.Group.Nodes.Builder nb = new StorDistributionConfig.Group.Nodes.Builder(); nb.index(node.getDistributionKey()); @@ -384,11 +382,7 @@ public class StorageGroup { Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostsPerGroup = new LinkedHashMap<>(); for (Map.Entry<HostResource, ClusterMembership> entry : hostMapping.entrySet()) { Optional<ClusterSpec.Group> group = entry.getValue().cluster().group(); - Map<HostResource, ClusterMembership> hostsInGroup = hostsPerGroup.get(group); - if (hostsInGroup == null) { - hostsInGroup = new LinkedHashMap<>(); - hostsPerGroup.put(group, hostsInGroup); - } + Map<HostResource, ClusterMembership> hostsInGroup = hostsPerGroup.computeIfAbsent(group, k -> new LinkedHashMap<>()); hostsInGroup.put(entry.getKey(), entry.getValue()); } return hostsPerGroup; @@ -396,23 +390,15 @@ public class StorageGroup { } - private static class XmlNodeBuilder { - - private final ModelElement clusterElement; - private final ModelElement element; - - private XmlNodeBuilder(ModelElement clusterElement, ModelElement element) { - this.clusterElement = clusterElement; - this.element = element; - } + private record XmlNodeBuilder(ModelElement clusterElement, ModelElement element) { public StorageNode build(DeployState deployState, ContentCluster parent, StorageGroup storageGroup) { - StorageNode sNode = new StorageNode.Builder().build(deployState, parent.getStorageCluster(), element.getXml()); - PersistenceEngine provider = parent.getPersistence().create(deployState, sNode, storageGroup, element); - new Distributor.Builder(clusterElement, provider).build(deployState, parent.getDistributorNodes(), element.getXml()); - return sNode; - } - } + StorageNode sNode = new StorageNode.Builder().build(deployState, parent.getStorageCluster(), element.getXml()); + PersistenceEngine provider = parent.getPersistence().create(deployState, sNode, storageGroup, element); + new Distributor.Builder(clusterElement, provider).build(deployState, parent.getDistributorNodes(), element.getXml()); + return sNode; + } + } /** * Creates a content group builder from a group and/or nodes element. @@ -441,14 +427,13 @@ public class StorageGroup { childAsString(groupElement, VespaDomBuilder.VESPAMALLOC_DEBUG), childAsString(groupElement, VespaDomBuilder.VESPAMALLOC_DEBUG_STACKTRACE)); - List<GroupBuilder> subGroups = groupElement.isPresent() ? collectSubGroups(isHosted, group, groupElement.get()) - : List.of(); + List<GroupBuilder> subGroups = groupElement.map(modelElement -> collectSubGroups(isHosted, group, modelElement)).orElseGet(List::of); List<XmlNodeBuilder> explicitNodes = new ArrayList<>(); explicitNodes.addAll(collectExplicitNodes(groupElement)); explicitNodes.addAll(collectExplicitNodes(nodesElement)); - if (subGroups.size() > 0 && nodesElement.isPresent()) + if (!subGroups.isEmpty() && nodesElement.isPresent()) throw new IllegalArgumentException("A group can contain either explicit subgroups or a nodes specification, but not both."); Optional<NodesSpecification> nodeRequirement; @@ -474,12 +459,10 @@ public class StorageGroup { return Optional.ofNullable(element.get().childAsString(childTagName)); } private Optional<Long> childAsLong(Optional<ModelElement> element, String childTagName) { - if (element.isEmpty()) return Optional.empty(); - return Optional.ofNullable(element.get().childAsLong(childTagName)); + return element.map(modelElement -> modelElement.childAsLong(childTagName)); } private Optional<Boolean> childAsBoolean(Optional<ModelElement> element, String childTagName) { - if (element.isEmpty()) return Optional.empty(); - return Optional.ofNullable(element.get().childAsBoolean(childTagName)); + return element.map(modelElement -> modelElement.childAsBoolean(childTagName)); } private boolean booleanAttributeOr(Optional<ModelElement> element, String attributeName, boolean defaultValue) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java index 65e329ab622..8600b40d5a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import com.yahoo.vespa.model.content.engines.*; +import com.yahoo.vespa.model.content.engines.DummyPersistence; +import com.yahoo.vespa.model.content.engines.PersistenceEngine; +import com.yahoo.vespa.model.content.engines.ProtonEngine; /** * Creates the correct engine factory from XML. @@ -18,7 +20,7 @@ public class EngineFactoryBuilder { if (persistence.child("proton") != null) { return new ProtonEngine.Factory(c.getSearch()); } else if (persistence.child("dummy") != null) { - return new com.yahoo.vespa.model.content.engines.DummyPersistence.Factory(); + return new DummyPersistence.Factory(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java deleted file mode 100644 index 643a305f369..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import java.util.Map; -import java.util.TreeMap; - -/** - * Class representing a group of @link{SearchInterface} nodes and a set of @link{Dispatch} nodes. - * - * Each @link{Dispatch} has a reference to an instance of this class and use it when producing config. - * - * @author baldersheim - */ -public class DispatchGroup { - - private final Map<Integer, Map<Integer, SearchInterface>> searchers = new TreeMap<>(); - - final private IndexedSearchCluster sc; - - public DispatchGroup(IndexedSearchCluster sc) { - this.sc = sc; - } - - DispatchGroup addSearcher(SearchInterface search) { - Map<Integer, SearchInterface> rows = searchers.get(search.getNodeSpec().partitionId()); - if (rows == null) { - rows = new TreeMap<>(); - rows.put(search.getNodeSpec().groupIndex(), search); - searchers.put(search.getNodeSpec().partitionId(), rows); - } else { - if (rows.containsKey(search.getNodeSpec().groupIndex())) { - throw new IllegalArgumentException("Already contains a search node with row id '" + search.getNodeSpec().groupIndex() + "'"); - } - rows.put(search.getNodeSpec().groupIndex(), search); - } - return this; - } - - public Iterable getSearchersIterable() { - return new Iterable(searchers); - } - - public int getRowBits() { - return sc.getRowBits(); - } - - public int getNumPartitions() { - return searchers.size(); - } - - public boolean useFixedRowInDispatch() { - return sc.useFixedRowInDispatch(); - } - - public int getSearchableCopies() { return sc.getSearchableCopies(); } - - public int getRedundancy() { return sc.getRedundancy(); } - - static class Iterator implements java.util.Iterator<SearchInterface> { - - private java.util.Iterator<Map<Integer, SearchInterface>> it1; - private java.util.Iterator<SearchInterface> it2; - - Iterator(Map<Integer, Map<Integer, SearchInterface> > s) { - it1 = s.values().iterator(); - if (it1.hasNext()) { - it2 = it1.next().values().iterator(); - } - } - - @Override - public boolean hasNext() { - if (it2 == null) { - return false; - } - while (!it2.hasNext() && it1.hasNext()) { - it2 = it1.next().values().iterator(); - } - return it2.hasNext(); - } - - @Override - public SearchInterface next() { - return it2.next(); - } - - @Override - public void remove() { - throw new IllegalStateException("'remove' not implemented"); - } - } - - public static class Iterable implements java.lang.Iterable<SearchInterface> { - final Map<Integer, Map<Integer, SearchInterface> > searchers; - Iterable(Map<Integer, Map<Integer, SearchInterface> > searchers) { this.searchers = searchers; } - @Override - public java.util.Iterator<SearchInterface> iterator() { - return new Iterator(searchers); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index a71e72c0ef8..65a667feb4f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -50,10 +50,8 @@ public class IndexedSearchCluster extends SearchCluster private final List<DocumentDatabase> documentDbs = new LinkedList<>(); private final MultipleDocumentDatabasesConfigProducer documentDbsConfigProducer; - private int searchableCopies = 1; private int redundancy = 1; - private final DispatchGroup rootDispatch; private final List<SearchNode> searchNodes = new ArrayList<>(); private final DispatchTuning.DispatchPolicy defaultDispatchPolicy; private final double dispatchWarmup; @@ -72,7 +70,6 @@ public class IndexedSearchCluster extends SearchCluster super(parent, clusterName, index); indexingDocproc = new IndexingDocproc(); documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); - rootDispatch = new DispatchGroup(this); defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); dispatchWarmup = featureFlags.queryDispatchWarmup(); summaryDecodePolicy = featureFlags.summaryDecodePolicy(); @@ -83,11 +80,9 @@ public class IndexedSearchCluster extends SearchCluster public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } - public DispatchGroup getRootDispatch() { return rootDispatch; } public void addSearcher(SearchNode searcher) { searchNodes.add(searcher); - rootDispatch.addSearcher(searcher); } public List<SearchNode> getSearchNodes() { return Collections.unmodifiableList(searchNodes); } @@ -198,27 +193,6 @@ public class IndexedSearchCluster extends SearchCluster documentDbsConfigProducer.getConfig(builder); } - boolean useFixedRowInDispatch() { - for (SearchNode node : getSearchNodes()) { - if (node.getNodeSpec().groupIndex() > 0) { - return true; - } - } - return false; - } - - public int getSearchableCopies() { - return searchableCopies; - } - - public void setSearchableCopies(int searchableCopies) { - this.searchableCopies = searchableCopies; - } - - public int getRedundancy() { - return redundancy; - } - public void setRedundancy(int redundancy) { this.redundancy = redundancy; } @@ -258,7 +232,7 @@ public class IndexedSearchCluster extends SearchCluster if (tuning.dispatch.getMaxHitsPerPartition() != null) builder.maxHitsPerNode(tuning.dispatch.getMaxHitsPerPartition()); - builder.redundancy(rootDispatch.getRedundancy()); + builder.redundancy(redundancy); if (searchCoverage != null) { if (searchCoverage.getMinimum() != null) builder.minSearchCoverage(searchCoverage.getMinimum() * 100.0); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 1ad99404823..02f15ed06cd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.content.Redundancy; import static java.lang.Long.min; import static java.lang.Long.max; @@ -27,13 +28,16 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; + private final Redundancy redundancy; public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, + Redundancy redundancy) { this.resources = resources; this.threadsPerSearch = threadsPerSearch; this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.redundancy = redundancy; } @Override @@ -57,7 +61,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_STORE_ONLY; - builder.allocation.initialnumdocs(numDocs); + builder.allocation.initialnumdocs(numDocs/redundancy.readyCopies()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index c3de655af92..53da4d31488 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.ContentNode; +import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.content.ResourceLimits; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; @@ -69,6 +70,7 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private final double fractionOfMemoryReserved; + private final Redundancy.Provider redundancyProvider; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<SearchNode> { @@ -80,10 +82,11 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private final double fractionOfMemoryReserved; + private final Redundancy.Provider redundancyProvider; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Redundancy.Provider redundancyprovider) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -92,23 +95,26 @@ public class SearchNode extends AbstractService implements this.tuning = tuning; this.resourceLimits = resourceLimits; this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.redundancyProvider = redundancyprovider; } @Override - protected SearchNode doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { + protected SearchNode doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, + Element producerSpec) { return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, redundancyProvider, deployState.featureFlags()); } } public static SearchNode create(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, - Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, ModelContext.FeatureFlags featureFlags) { + Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, + boolean isHostedVespa, double fractionOfMemoryReserved, Redundancy.Provider redundancyProvider, + ModelContext.FeatureFlags featureFlags) { SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, redundancyProvider); if (featureFlags.loadCodeAsHugePages()) { node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true); } @@ -121,7 +127,7 @@ public class SearchNode extends AbstractService implements private SearchNode(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Redundancy.Provider redundancyProvider) { super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; @@ -138,6 +144,7 @@ public class SearchNode extends AbstractService implements // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; + this.redundancyProvider = redundancyProvider; setPropertiesElastic(clusterName, distributionKey); addEnvironmentVariable("OMP_NUM_THREADS", 1); } @@ -279,7 +286,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - fractionOfMemoryReserved); + fractionOfMemoryReserved, redundancyProvider.redundancy()); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 285911549d7..74f6b5b003c 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -105,8 +105,6 @@ ModelEvaluation = element model-evaluation { element interop-threads { xsd:nonNegativeInteger }? & element execution-mode { string "sequential" | string "parallel" }? & element gpu-device { - # TODO(mpolden): Remove this after 2023-02-01 - attribute required { xsd:boolean }? & xsd:nonNegativeInteger }? }* diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index d47ce81eaac..87783c1ee20 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -122,7 +122,6 @@ Perf = element perf { } Prod = element prod { - attribute global-service-id { text }? & attribute athenz-service { xsd:string }? & attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & |