diff options
20 files changed, 238 insertions, 340 deletions
diff --git a/client/go/script-utils/startcbinary/common_env.go b/client/go/script-utils/startcbinary/common_env.go index eadb0aafb1b..eee7500f5d2 100644 --- a/client/go/script-utils/startcbinary/common_env.go +++ b/client/go/script-utils/startcbinary/common_env.go @@ -70,7 +70,7 @@ func (spec *ProgSpec) configureUseMadvise() { func (spec *ProgSpec) configurePath() { // Prefer newer gdb and pstack: - spec.prependPath("/opt/rh/gcc-toolset-11/root/usr/bin") + spec.prependPath("/opt/rh/gcc-toolset-12/root/usr/bin") // Maven is needed for tester applications: spec.prependPath(vespa.FindHome() + "/local/maven/bin") spec.prependPath(vespa.FindHome() + "/bin64") diff --git a/client/go/vespa/load_env.go b/client/go/vespa/load_env.go index 5c216c12c31..3c73f7aaf53 100644 --- a/client/go/vespa/load_env.go +++ b/client/go/vespa/load_env.go @@ -17,7 +17,7 @@ import ( ) const ( - CURRENT_GCC_TOOLSET = "/opt/rh/gcc-toolset-11/root/usr/bin" + CURRENT_GCC_TOOLSET = "/opt/rh/gcc-toolset-12/root/usr/bin" ) // backwards-compatible parsing of default-env.txt @@ -290,7 +290,7 @@ func ensureGoodPath(receiver loadEnvReceiver) { builder.appendPath(FindHome() + "/bin") builder.appendPath(FindHome() + "/bin64") // Prefer newer gdb and pstack: - builder.appendPath("/opt/rh/gcc-toolset-11/root/usr/bin") + builder.appendPath("/opt/rh/gcc-toolset-12/root/usr/bin") // how to find the "java" program? if javaHome := os.Getenv(envvars.JAVA_HOME); javaHome != "" { builder.appendPath(javaHome + "/bin") diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 66073b1d7d1..d8232987524 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -243,18 +243,10 @@ public class VespaMetricSet { metrics.add(new Metric("cluster-controller.stopping.count.last")); metrics.add(new Metric("cluster-controller.up.count.last")); metrics.add(new Metric("cluster-controller.cluster-state-change.count")); - metrics.add(new Metric("cluster-controller.busy-tick-time-ms.last")); - metrics.add(new Metric("cluster-controller.busy-tick-time-ms.max")); - metrics.add(new Metric("cluster-controller.busy-tick-time-ms.sum")); - metrics.add(new Metric("cluster-controller.busy-tick-time-ms.count")); - metrics.add(new Metric("cluster-controller.idle-tick-time-ms.last")); - metrics.add(new Metric("cluster-controller.idle-tick-time-ms.max")); - metrics.add(new Metric("cluster-controller.idle-tick-time-ms.sum")); - metrics.add(new Metric("cluster-controller.idle-tick-time-ms.count")); - - metrics.add(new Metric("cluster-controller.work-ms.last")); - metrics.add(new Metric("cluster-controller.work-ms.sum")); - metrics.add(new Metric("cluster-controller.work-ms.count")); + addMetric(metrics, "cluster-controller.busy-tick-time-ms", List.of("last", "max", "sum", "count")); + addMetric(metrics, "cluster-controller.idle-tick-time-ms", List.of("last", "max", "sum", "count")); + + addMetric(metrics, "cluster-controller.work-ms", List.of("last", "sum", "count")); metrics.add(new Metric("cluster-controller.is-master.last")); metrics.add(new Metric("cluster-controller.remote-task-queue.size.last")); @@ -262,12 +254,9 @@ public class VespaMetricSet { // DO NOT RELY ON THIS METRIC YET. metrics.add(new Metric("cluster-controller.node-event.count")); - metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.last")); - metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.max")); - metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.last")); - metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.max")); - metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.last")); - metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.max")); + addMetric(metrics, "cluster-controller.resource_usage.nodes_above_limit", List.of("last", "max")); + addMetric(metrics, "cluster-controller.resource_usage.max_memory_utilization", List.of("last", "max")); + addMetric(metrics, "cluster-controller.resource_usage.max_disk_utilization", List.of("last", "max")); metrics.add(new Metric("cluster-controller.resource_usage.disk_limit.last")); metrics.add(new Metric("cluster-controller.resource_usage.memory_limit.last")); @@ -356,32 +345,16 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.heart_beat_age.last")); metrics.add(new Metric("content.proton.transport.query.count.rate")); metrics.add(new Metric("content.proton.docsum.docs.rate")); - metrics.add(new Metric("content.proton.docsum.latency.max")); - metrics.add(new Metric("content.proton.docsum.latency.sum")); - metrics.add(new Metric("content.proton.docsum.latency.count")); - metrics.add(new Metric("content.proton.transport.query.latency.max")); - metrics.add(new Metric("content.proton.transport.query.latency.sum")); - metrics.add(new Metric("content.proton.transport.query.latency.count")); + addMetric(metrics, "content.proton.docsum.latency", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.transport.query.latency", List.of("max", "sum", "count")); // Search protocol - metrics.add(new Metric("content.proton.search_protocol.query.latency.max")); - metrics.add(new Metric("content.proton.search_protocol.query.latency.sum")); - metrics.add(new Metric("content.proton.search_protocol.query.latency.count")); - metrics.add(new Metric("content.proton.search_protocol.query.request_size.max")); - metrics.add(new Metric("content.proton.search_protocol.query.request_size.sum")); - metrics.add(new Metric("content.proton.search_protocol.query.request_size.count")); - metrics.add(new Metric("content.proton.search_protocol.query.reply_size.max")); - metrics.add(new Metric("content.proton.search_protocol.query.reply_size.sum")); - metrics.add(new Metric("content.proton.search_protocol.query.reply_size.count")); - metrics.add(new Metric("content.proton.search_protocol.docsum.latency.max")); - metrics.add(new Metric("content.proton.search_protocol.docsum.latency.sum")); - metrics.add(new Metric("content.proton.search_protocol.docsum.latency.count")); - metrics.add(new Metric("content.proton.search_protocol.docsum.request_size.max")); - metrics.add(new Metric("content.proton.search_protocol.docsum.request_size.sum")); - metrics.add(new Metric("content.proton.search_protocol.docsum.request_size.count")); - metrics.add(new Metric("content.proton.search_protocol.docsum.reply_size.max")); - metrics.add(new Metric("content.proton.search_protocol.docsum.reply_size.sum")); - metrics.add(new Metric("content.proton.search_protocol.docsum.reply_size.count")); + addMetric(metrics, "content.proton.search_protocol.query.latency", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.search_protocol.query.request_size", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.search_protocol.query.reply_size", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.search_protocol.docsum.latency", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.search_protocol.docsum.request_size", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.search_protocol.docsum.reply_size", List.of("max", "sum", "count")); metrics.add(new Metric("content.proton.search_protocol.docsum.requested_documents.count")); // Executors shared between all document dbs @@ -449,21 +422,11 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.max")); // CPU util - metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.max")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.sum")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.count")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.max")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.sum")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.count")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.max")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.sum")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.count")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.max")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.sum")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.count")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.max")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.sum")); - metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.count")); + addMetric(metrics, "content.proton.resource_usage.cpu_util.setup", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.resource_usage.cpu_util.read", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.resource_usage.cpu_util.write", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.resource_usage.cpu_util.compact", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.resource_usage.cpu_util.other", List.of("max", "sum", "count")); // transaction log metrics.add(new Metric("content.proton.transactionlog.entries.average")); @@ -522,44 +485,22 @@ public class VespaMetricSet { // matching metrics.add(new Metric("content.proton.documentdb.matching.queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.soft_doomed_queries.rate")); - metrics.add(new Metric("content.proton.documentdb.matching.query_latency.max")); - metrics.add(new Metric("content.proton.documentdb.matching.query_latency.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.query_latency.count")); - metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count")); - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.count")); + addMetric(metrics, "content.proton.documentdb.matching.query_latency", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.documentdb.matching.query_setup_time", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.documentdb.matching.docs_matched", List.of("rate", "count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doom_factor.min")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doom_factor.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doom_factor.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doom_factor.count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.grouping_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.grouping_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.grouping_time.count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.rate")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.count")); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.soft_doom_factor", List.of("min", "max", "sum", "count")); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.query_latency", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.query_setup_time", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.grouping_time", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.rerank_time", List.of("max", "sum", "count")); + addMetric(metrics, "content.proton.documentdb.matching.rank_profile.docs_matched", List.of("rate", "count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.limited_queries.rate")); // feeding - metrics.add(new Metric("content.proton.documentdb.feeding.commit.operations.max")); - metrics.add(new Metric("content.proton.documentdb.feeding.commit.operations.sum")); - metrics.add(new Metric("content.proton.documentdb.feeding.commit.operations.count")); - metrics.add(new Metric("content.proton.documentdb.feeding.commit.operations.rate")); - metrics.add(new Metric("content.proton.documentdb.feeding.commit.latency.max")); - metrics.add(new Metric("content.proton.documentdb.feeding.commit.latency.sum")); - metrics.add(new Metric("content.proton.documentdb.feeding.commit.latency.count")); + addMetric(metrics, "content.proton.documentdb.feeding.commit.operations", List.of("max", "sum", "count", "rate")); + addMetric(metrics, "content.proton.documentdb.feeding.commit.latency", List.of("max", "sum", "count")); return metrics; } @@ -573,61 +514,27 @@ public class VespaMetricSet { metrics.add(new Metric("vds.datastored.alldisks.buckets.average")); metrics.add(new Metric("vds.datastored.alldisks.docs.average")); metrics.add(new Metric("vds.datastored.alldisks.bytes.average")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.max")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.count")); - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.max")); - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum")); - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.count")); - metrics.add(new Metric("vds.visitor.allthreads.queuesize.max")); - metrics.add(new Metric("vds.visitor.allthreads.queuesize.sum")); - metrics.add(new Metric("vds.visitor.allthreads.queuesize.count")); + addMetric(metrics, "vds.visitor.allthreads.averagevisitorlifetime", List.of("max", "sum", "count")); + addMetric(metrics, "vds.visitor.allthreads.averagequeuewait", List.of("max", "sum", "count")); + addMetric(metrics, "vds.visitor.allthreads.queuesize", List.of("max", "sum", "count")); metrics.add(new Metric("vds.visitor.allthreads.completed.rate")); metrics.add(new Metric("vds.visitor.allthreads.created.rate")); metrics.add(new Metric("vds.visitor.allthreads.failed.rate")); - metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.max")); - metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum")); - metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.count")); - metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.max")); - metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum")); - metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.count")); - - metrics.add(new Metric("vds.filestor.queuesize.max")); - metrics.add(new Metric("vds.filestor.queuesize.sum")); - metrics.add(new Metric("vds.filestor.queuesize.count")); - metrics.add(new Metric("vds.filestor.averagequeuewait.max")); - metrics.add(new Metric("vds.filestor.averagequeuewait.sum")); - metrics.add(new Metric("vds.filestor.averagequeuewait.count")); - metrics.add(new Metric("vds.filestor.active_operations.size.max")); - metrics.add(new Metric("vds.filestor.active_operations.size.sum")); - metrics.add(new Metric("vds.filestor.active_operations.size.count")); - metrics.add(new Metric("vds.filestor.active_operations.latency.max")); - metrics.add(new Metric("vds.filestor.active_operations.latency.sum")); - metrics.add(new Metric("vds.filestor.active_operations.latency.count")); - metrics.add(new Metric("vds.filestor.throttle_window_size.max")); - metrics.add(new Metric("vds.filestor.throttle_window_size.sum")); - metrics.add(new Metric("vds.filestor.throttle_window_size.count")); - metrics.add(new Metric("vds.filestor.throttle_waiting_threads.max")); - metrics.add(new Metric("vds.filestor.throttle_waiting_threads.sum")); - metrics.add(new Metric("vds.filestor.throttle_waiting_threads.count")); - metrics.add(new Metric("vds.filestor.throttle_active_tokens.max")); - metrics.add(new Metric("vds.filestor.throttle_active_tokens.sum")); - metrics.add(new Metric("vds.filestor.throttle_active_tokens.count")); - metrics.add(new Metric("vds.filestor.allthreads.mergemetadatareadlatency.max")); - metrics.add(new Metric("vds.filestor.allthreads.mergemetadatareadlatency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.mergemetadatareadlatency.count")); - metrics.add(new Metric("vds.filestor.allthreads.mergedatareadlatency.max")); - metrics.add(new Metric("vds.filestor.allthreads.mergedatareadlatency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.mergedatareadlatency.count")); - metrics.add(new Metric("vds.filestor.allthreads.mergedatawritelatency.max")); - metrics.add(new Metric("vds.filestor.allthreads.mergedatawritelatency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.mergedatawritelatency.count")); - metrics.add(new Metric("vds.filestor.allthreads.put_latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.put_latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.put_latency.count")); - metrics.add(new Metric("vds.filestor.allthreads.remove_latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.remove_latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.remove_latency.count")); + addMetric(metrics, "vds.visitor.allthreads.averagemessagesendtime", List.of("max", "sum", "count")); + addMetric(metrics, "vds.visitor.allthreads.averageprocessingtime", List.of("max", "sum", "count")); + + addMetric(metrics, "vds.filestor.queuesize", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.averagequeuewait", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.active_operations.size", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.active_operations.latency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.throttle_window_size", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.throttle_waiting_threads", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.throttle_active_tokens", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.mergemetadatareadlatency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.mergedatareadlatency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.mergedatawritelatency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.put_latency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.remove_latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.filestor.allstripes.throttled_rpc_direct_dispatches.rate")); metrics.add(new Metric("vds.filestor.allstripes.throttled_persistence_thread_polls.rate")); metrics.add(new Metric("vds.filestor.allstripes.timeouts_waiting_for_throttle_token.rate")); @@ -635,57 +542,33 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.allthreads.put.count.rate")); metrics.add(new Metric("vds.filestor.allthreads.put.failed.rate")); metrics.add(new Metric("vds.filestor.allthreads.put.test_and_set_failed.rate")); - metrics.add(new Metric("vds.filestor.allthreads.put.latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.put.latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.put.latency.count")); - metrics.add(new Metric("vds.filestor.allthreads.put.request_size.max")); - metrics.add(new Metric("vds.filestor.allthreads.put.request_size.sum")); - metrics.add(new Metric("vds.filestor.allthreads.put.request_size.count")); + addMetric(metrics, "vds.filestor.allthreads.put.latency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.put.request_size", List.of("max", "sum", "count")); metrics.add(new Metric("vds.filestor.allthreads.remove.count.rate")); metrics.add(new Metric("vds.filestor.allthreads.remove.failed.rate")); metrics.add(new Metric("vds.filestor.allthreads.remove.test_and_set_failed.rate")); - metrics.add(new Metric("vds.filestor.allthreads.remove.latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.remove.latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.remove.latency.count")); - metrics.add(new Metric("vds.filestor.allthreads.remove.request_size.max")); - metrics.add(new Metric("vds.filestor.allthreads.remove.request_size.sum")); - metrics.add(new Metric("vds.filestor.allthreads.remove.request_size.count")); + addMetric(metrics, "vds.filestor.allthreads.remove.latency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.remove.request_size", List.of("max", "sum", "count")); metrics.add(new Metric("vds.filestor.allthreads.get.count.rate")); metrics.add(new Metric("vds.filestor.allthreads.get.failed.rate")); - metrics.add(new Metric("vds.filestor.allthreads.get.latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.get.latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.get.latency.count")); - metrics.add(new Metric("vds.filestor.allthreads.get.request_size.max")); - metrics.add(new Metric("vds.filestor.allthreads.get.request_size.sum")); - metrics.add(new Metric("vds.filestor.allthreads.get.request_size.count")); + addMetric(metrics, "vds.filestor.allthreads.get.latency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.get.request_size", List.of("max", "sum", "count")); metrics.add(new Metric("vds.filestor.allthreads.update.count.rate")); metrics.add(new Metric("vds.filestor.allthreads.update.failed.rate")); metrics.add(new Metric("vds.filestor.allthreads.update.test_and_set_failed.rate")); - metrics.add(new Metric("vds.filestor.allthreads.update.latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.update.latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.update.latency.count")); - metrics.add(new Metric("vds.filestor.allthreads.update.request_size.max")); - metrics.add(new Metric("vds.filestor.allthreads.update.request_size.sum")); - metrics.add(new Metric("vds.filestor.allthreads.update.request_size.count")); - metrics.add(new Metric("vds.filestor.allthreads.createiterator.latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.createiterator.latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.createiterator.latency.count")); + addMetric(metrics, "vds.filestor.allthreads.update.latency", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.update.request_size", List.of("max", "sum", "count")); + addMetric(metrics, "vds.filestor.allthreads.createiterator.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.filestor.allthreads.createiterator.count.rate")); metrics.add(new Metric("vds.filestor.allthreads.visit.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.visit.latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.visit.latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.visit.latency.count")); + addMetric(metrics, "vds.filestor.allthreads.visit.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.filestor.allthreads.remove_location.count.rate")); - metrics.add(new Metric("vds.filestor.allthreads.remove_location.latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.remove_location.latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.remove_location.latency.count")); + addMetric(metrics, "vds.filestor.allthreads.remove_location.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.filestor.allthreads.splitbuckets.count.rate")); metrics.add(new Metric("vds.filestor.allthreads.joinbuckets.count.rate")); metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.count.rate")); metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.failed.rate")); - metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.latency.max")); - metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.latency.sum")); - metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.latency.count")); + addMetric(metrics, "vds.filestor.allthreads.deletebuckets.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.filestor.allthreads.setbucketstates.count.rate")); return metrics; } @@ -722,12 +605,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.idealstate.garbage_collection.done_ok.rate")); metrics.add(new Metric("vds.idealstate.garbage_collection.done_failed.rate")); metrics.add(new Metric("vds.idealstate.garbage_collection.pending.average")); - metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.count")); - metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.rate")); + addMetric(metrics, "vds.idealstate.garbage_collection.documents_removed", List.of("count", "rate")); - metrics.add(new Metric("vds.distributor.puts.latency.max")); - metrics.add(new Metric("vds.distributor.puts.latency.sum")); - metrics.add(new Metric("vds.distributor.puts.latency.count")); + addMetric(metrics, "vds.distributor.puts.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.distributor.puts.ok.rate")); metrics.add(new Metric("vds.distributor.puts.failures.total.rate")); metrics.add(new Metric("vds.distributor.puts.failures.notfound.rate")); @@ -741,17 +621,13 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.puts.failures.timeout.rate")); metrics.add(new Metric("vds.distributor.puts.failures.busy.rate")); metrics.add(new Metric("vds.distributor.puts.failures.inconsistent_bucket.rate")); - metrics.add(new Metric("vds.distributor.removes.latency.max")); - metrics.add(new Metric("vds.distributor.removes.latency.sum")); - metrics.add(new Metric("vds.distributor.removes.latency.count")); + addMetric(metrics, "vds.distributor.removes.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.distributor.removes.ok.rate")); metrics.add(new Metric("vds.distributor.removes.failures.total.rate")); metrics.add(new Metric("vds.distributor.removes.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.removes.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.removes.failures.concurrent_mutations.rate")); - metrics.add(new Metric("vds.distributor.updates.latency.max")); - metrics.add(new Metric("vds.distributor.updates.latency.sum")); - metrics.add(new Metric("vds.distributor.updates.latency.count")); + addMetric(metrics, "vds.distributor.updates.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.distributor.updates.ok.rate")); metrics.add(new Metric("vds.distributor.updates.failures.total.rate")); metrics.add(new Metric("vds.distributor.updates.failures.notfound.rate")); @@ -760,15 +636,11 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.updates.diverging_timestamp_updates.rate")); metrics.add(new Metric("vds.distributor.removelocations.ok.rate")); metrics.add(new Metric("vds.distributor.removelocations.failures.total.rate")); - metrics.add(new Metric("vds.distributor.gets.latency.max")); - metrics.add(new Metric("vds.distributor.gets.latency.sum")); - metrics.add(new Metric("vds.distributor.gets.latency.count")); + addMetric(metrics, "vds.distributor.gets.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.distributor.gets.ok.rate")); metrics.add(new Metric("vds.distributor.gets.failures.total.rate")); metrics.add(new Metric("vds.distributor.gets.failures.notfound.rate")); - metrics.add(new Metric("vds.distributor.visitor.latency.max")); - metrics.add(new Metric("vds.distributor.visitor.latency.sum")); - metrics.add(new Metric("vds.distributor.visitor.latency.count")); + addMetric(metrics, "vds.distributor.visitor.latency", List.of("max", "sum", "count")); metrics.add(new Metric("vds.distributor.visitor.ok.rate")); metrics.add(new Metric("vds.distributor.visitor.failures.total.rate")); metrics.add(new Metric("vds.distributor.visitor.failures.notready.rate")); @@ -786,15 +658,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.bouncer.clock_skew_aborts.count")); - metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.max")); - metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.sum")); - metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.count")); - metrics.add(new Metric("vds.mergethrottler.queuesize.max")); - metrics.add(new Metric("vds.mergethrottler.queuesize.sum")); - metrics.add(new Metric("vds.mergethrottler.queuesize.count")); - metrics.add(new Metric("vds.mergethrottler.active_window_size.max")); - metrics.add(new Metric("vds.mergethrottler.active_window_size.sum")); - metrics.add(new Metric("vds.mergethrottler.active_window_size.count")); + addMetric(metrics, "vds.mergethrottler.averagequeuewaitingtime", List.of("max", "sum", "count")); + addMetric(metrics, "vds.mergethrottler.queuesize", List.of("max", "sum", "count")); + addMetric(metrics, "vds.mergethrottler.active_window_size", List.of("max", "sum", "count")); metrics.add(new Metric("vds.mergethrottler.bounced_due_to_back_pressure.rate")); metrics.add(new Metric("vds.mergethrottler.locallyexecutedmerges.ok.rate")); metrics.add(new Metric("vds.mergethrottler.mergechains.ok.rate")); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index 41b69d2376d..6753fed2c5a 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -26,8 +26,8 @@ class MetricDefinitions { static final String CONNECTION_DURATION_MAX = "serverConnectionDurationMax"; static final String CONNECTION_DURATION_MEAN = "serverConnectionDurationMean"; static final String CONNECTION_DURATION_STD_DEV = "serverConnectionDurationStdDev"; - static final String NUM_PREMATURELY_CLOSED_CONNECTIONS = "jdisc.http.request.prematurely_closed"; - static final String REQUESTS_PER_CONNECTION = "jdisc.http.request.requests_per_connection"; + static final String NUM_PREMATURELY_CLOSED_CONNECTIONS = ContainerMetrics.JDISC_HTTP_REQUEST_PREMATURELY_CLOSED.baseName(); + static final String REQUESTS_PER_CONNECTION = ContainerMetrics.JDISC_HTTP_REQUEST_REQUESTS_PER_CONNECTION.baseName(); static final String NUM_BYTES_RECEIVED = ContainerMetrics.SERVER_BYTES_RECEIVED.baseName(); static final String NUM_BYTES_SENT = ContainerMetrics.SERVER_BYTES_SENT.baseName(); @@ -35,7 +35,7 @@ class MetricDefinitions { static final String NUM_CONNECTIONS = ContainerMetrics.SERVER_NUM_CONNECTIONS.baseName(); /* For historical reasons, these are all aliases for the same metric. 'jdisc.http' should ideally be the only one. */ - static final String JDISC_HTTP_REQUESTS = "jdisc.http.requests"; + static final String JDISC_HTTP_REQUESTS = ContainerMetrics.JDISC_HTTP_REQUESTS.baseName(); static final String NUM_REQUESTS = "serverNumRequests"; static final String NUM_SUCCESSFUL_RESPONSES = "serverNumSuccessfulResponses"; @@ -55,16 +55,16 @@ class MetricDefinitions { static final String STARTED_MILLIS = "serverStartedMillis"; - static final String URI_LENGTH = "jdisc.http.request.uri_length"; - static final String CONTENT_SIZE = "jdisc.http.request.content_size"; + static final String URI_LENGTH = ContainerMetrics.JDISC_HTTP_REQUEST_URI_LENGTH.baseName(); + static final String CONTENT_SIZE = ContainerMetrics.JDISC_HTTP_REQUEST_CONTENT_SIZE.baseName(); - static final String SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.missing_client_cert"; - static final String SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.expired_client_cert"; - static final String SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.invalid_client_cert"; - static final String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS = "jdisc.http.ssl.handshake.failure.incompatible_protocols"; - static final String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS = "jdisc.http.ssl.handshake.failure.incompatible_ciphers"; - static final String SSL_HANDSHAKE_FAILURE_UNKNOWN = "jdisc.http.ssl.handshake.failure.unknown"; - static final String SSL_HANDSHAKE_FAILURE_CONNECTION_CLOSED = "jdisc.http.ssl.handshake.failure.connection_closed"; + static final String SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT = ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT.baseName(); + static final String SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT = ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT.baseName(); + static final String SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT = ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT.baseName(); + static final String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS = ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS.baseName(); + static final String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS = ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CHIFERS.baseName(); + static final String SSL_HANDSHAKE_FAILURE_CONNECTION_CLOSED = ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_CONNECTION_CLOSED.baseName(); + static final String SSL_HANDSHAKE_FAILURE_UNKNOWN = ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_UNKNOWN.baseName(); static final String JETTY_THREADPOOL_MAX_THREADS = ContainerMetrics.JETTY_THREADPOOL_MAX_THREADS.baseName(); static final String JETTY_THREADPOOL_MIN_THREADS = ContainerMetrics.JETTY_THREADPOOL_MIN_THREADS.baseName(); @@ -74,10 +74,10 @@ class MetricDefinitions { static final String JETTY_THREADPOOL_TOTAL_THREADS = ContainerMetrics.JETTY_THREADPOOL_TOTAL_THREADS.baseName(); static final String JETTY_THREADPOOL_QUEUE_SIZE = ContainerMetrics.JETTY_THREADPOOL_QUEUE_SIZE.baseName(); - static final String FILTERING_REQUEST_HANDLED = "jdisc.http.filtering.request.handled"; - static final String FILTERING_REQUEST_UNHANDLED = "jdisc.http.filtering.request.unhandled"; - static final String FILTERING_RESPONSE_HANDLED = "jdisc.http.filtering.response.handled"; - static final String FILTERING_RESPONSE_UNHANDLED = "jdisc.http.filtering.response.unhandled"; + static final String FILTERING_REQUEST_HANDLED = ContainerMetrics.JDISC_HTTP_FILTERING_REQUEST_HANDLED.baseName(); + static final String FILTERING_REQUEST_UNHANDLED = ContainerMetrics.JDISC_HTTP_FILTERING_REQUEST_UNHANDLED.baseName(); + static final String FILTERING_RESPONSE_HANDLED = ContainerMetrics.JDISC_HTTP_FILTERING_RESPONSE_HANDLED.baseName(); + static final String FILTERING_RESPONSE_UNHANDLED = ContainerMetrics.JDISC_HTTP_FILTERING_RESPONSE_UNHANDLED.baseName(); private MetricDefinitions() {} } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java index da296361535..6adcd1c1985 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.dns; import ai.vespa.http.DomainName; -import com.yahoo.config.provision.zone.ZoneId; import java.util.Objects; @@ -20,8 +19,8 @@ public final class WeightedAliasTarget extends AliasTarget { private final long weight; - public WeightedAliasTarget(DomainName name, String dnsZone, ZoneId zone, long weight) { - super(name, dnsZone, zone.value()); + public WeightedAliasTarget(DomainName name, String dnsZone, String id, long weight) { + super(name, dnsZone, id); this.weight = weight; if (weight < 0) throw new IllegalArgumentException("Weight cannot be negative"); } @@ -65,8 +64,7 @@ public final class WeightedAliasTarget extends AliasTarget { if (!TARGET_TYPE.equals(parts[0])) { throw new IllegalArgumentException("Unexpected type '" + parts[0] + "'"); } - return new WeightedAliasTarget(DomainName.of(parts[1]), parts[2], ZoneId.from(parts[3]), - Long.parseLong(parts[4])); + return new WeightedAliasTarget(DomainName.of(parts[1]), parts[2], parts[3], Long.parseLong(parts[4])); } } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTargetTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTargetTest.java index 9cbba8107ef..c859cc020ab 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTargetTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTargetTest.java @@ -19,7 +19,7 @@ public class AliasTargetTest { void packing() { List<AliasTarget> tests = List.of( new LatencyAliasTarget(HostName.of("foo.example.com"), "dns-zone-1", ZoneId.from("prod.us-north-1")), - new WeightedAliasTarget(HostName.of("bar.example.com"), "dns-zone-2", ZoneId.from("prod.us-north-2"), 50) + new WeightedAliasTarget(HostName.of("bar.example.com"), "dns-zone-2", "prod.us-north-2", 50) ); for (var target : tests) { AliasTarget unpacked = AliasTarget.unpack(target.pack()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 03db932dbb3..25ad7da98e7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -233,7 +233,7 @@ public class RoutingPolicies { if (policy.canonicalName().isPresent()) { var weightedTarget = new WeightedAliasTarget( - policy.canonicalName().get(), policy.dnsZone().get(), policy.id().zone(), weight); + policy.canonicalName().get(), policy.dnsZone().get(), policy.id().zone().value(), weight); regionEndpoint.add(weightedTarget); } else { var weightedTarget = new WeightedDirectTarget( @@ -655,7 +655,7 @@ public class RoutingPolicies { return new Target(Record.Type.DIRECT, wt.recordData(), wt); } var wt = new WeightedAliasTarget(policy.canonicalName().get(), policy.dnsZone().get(), - endpointTarget.deployment().zoneId(), endpointTarget.weight()); + endpointTarget.deployment().zoneId().value(), endpointTarget.weight()); return new Target(Record.Type.ALIAS, RecordData.fqdn(wt.name().value()), wt); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index db96c265363..a049b614e25 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1020,7 +1020,7 @@ public class ControllerTest { new Record(Record.Type.ALIAS, RecordName.from("application.tenant.us-east-3-w.vespa.oath.cloud"), new WeightedAliasTarget(HostName.of("lb-0--tenant.application.default--prod.us-east-3"), - "dns-zone-1", ZoneId.from("prod.us-east-3"), 1).pack()), + "dns-zone-1", "prod.us-east-3", 1).pack()), // The 'east' global endpoint, pointing to the weighted record for zone 2's region new Record(Record.Type.ALIAS, diff --git a/dist/vespa.spec b/dist/vespa.spec index 2f45cf0ffa1..4c9d7c5df72 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -48,17 +48,17 @@ BuildRequires: epel-release %if 0%{?el8} %global _centos_stream %(grep -qs '^NAME="CentOS Stream"' /etc/os-release && echo 1 || echo 0) %if 0%{?_centos_stream} -BuildRequires: gcc-toolset-11-gcc-c++ -BuildRequires: gcc-toolset-11-binutils -BuildRequires: gcc-toolset-11-libasan-devel -BuildRequires: gcc-toolset-11-libatomic-devel -%define _devtoolset_enable /opt/rh/gcc-toolset-11/enable +BuildRequires: gcc-toolset-12-gcc-c++ +BuildRequires: gcc-toolset-12-binutils +BuildRequires: gcc-toolset-12-libasan-devel +BuildRequires: gcc-toolset-12-libatomic-devel +%define _devtoolset_enable /opt/rh/gcc-toolset-12/enable %else -BuildRequires: gcc-toolset-11-gcc-c++ -BuildRequires: gcc-toolset-11-binutils -BuildRequires: gcc-toolset-11-libasan-devel -BuildRequires: gcc-toolset-11-libatomic-devel -%define _devtoolset_enable /opt/rh/gcc-toolset-11/enable +BuildRequires: gcc-toolset-12-gcc-c++ +BuildRequires: gcc-toolset-12-binutils +BuildRequires: gcc-toolset-12-libasan-devel +BuildRequires: gcc-toolset-12-libatomic-devel +%define _devtoolset_enable /opt/rh/gcc-toolset-12/enable %endif BuildRequires: maven BuildRequires: maven-openjdk17 diff --git a/metrics/src/vespa/metrics/metricvalueset.h b/metrics/src/vespa/metrics/metricvalueset.h index b38cab4bfac..f94485c685e 100644 --- a/metrics/src/vespa/metrics/metricvalueset.h +++ b/metrics/src/vespa/metrics/metricvalueset.h @@ -29,6 +29,7 @@ #include <vector> #include <atomic> +#include <array> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/jsonstream.h> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index cfb2d240bfa..28773767d24 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -28,7 +28,7 @@ public class CoreCollector { private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("(?m)^Core was generated by `(?<path>.*?)'\\."); private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'"); private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'"); - static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-11/root/bin/gdb"; + static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-12/root/bin/gdb"; private final ContainerOperations container; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index 8f251bf72c5..4fa18c71da0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -95,7 +95,7 @@ public class CoreCollectorTest { fail("Expected not to be able to get bin path"); } catch (RuntimeException e) { assertEquals("Failed to extract binary path from GDB, result: exit status 1, output 'Error 123', command: " + - "[/opt/rh/gcc-toolset-11/root/bin/gdb, -n, -batch, -core, /tmp/core.1234]", e.getMessage()); + "[/opt/rh/gcc-toolset-12/root/bin/gdb, -n, -batch, -core, /tmp/core.1234]", e.getMessage()); } } diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index 1a9288c007f..a0e205d2597 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -6,7 +6,7 @@ set -e readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd )" readonly NUM_THREADS=$(( $(nproc) + 2 )) -source /etc/profile.d/enable-gcc-toolset-11.sh +source /etc/profile.d/enable-gcc-toolset-12.sh export MALLOC_ARENA_MAX=1 export MAVEN_OPTS="-Xss1m -Xms128m -Xmx2g" diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index e340427c83a..0463568213b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -11,6 +11,7 @@ #include <vespa/searchlib/fef/rank_program.h> #include <vespa/searchlib/queryeval/multibitvectoriterator.h> #include <vespa/searchlib/queryeval/andnotsearch.h> +#include <vespa/searchlib/queryeval/profiled_iterator.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> @@ -19,15 +20,16 @@ LOG_SETUP(".proton.matching.match_thread"); namespace proton::matching { -using search::queryeval::SearchIterator; +using search::attribute::AttributeOperation; using search::fef::BlueprintResolver; -using search::fef::MatchData; -using search::fef::RankProgram; using search::fef::FeatureResolver; using search::fef::LazyValue; +using search::fef::MatchData; +using search::fef::RankProgram; using search::queryeval::HitCollector; +using search::queryeval::ProfiledIterator; +using search::queryeval::SearchIterator; using search::queryeval::SortedHitSequence; -using search::attribute::AttributeOperation; namespace { @@ -286,6 +288,10 @@ MatchThread::findMatches(MatchTools &tools) tools.search().asSlime(inserter); } } + if (match_profiler) { + tools.give_back_search(ProfiledIterator::profile(*match_profiler, tools.borrow_search())); + tools.tag_search_as_changed(); + } HitCollector hits(matchParams.numDocs, matchParams.arraySize); trace->addEvent(4, "Start match and first phase rank"); match_loop_helper(tools, hits); @@ -417,11 +423,13 @@ MatchThread::MatchThread(size_t thread_id_in, wait_time_s(0.0), match_with_ranking(mtf.has_first_phase_rank() && mp.save_rank_scores()), trace(std::make_unique<Trace>(relativeTime, traceLevel, profileDepth)), + match_profiler(), first_phase_profiler(), second_phase_profiler(), my_issues() { if ((traceLevel > 0) && (profileDepth > 0)) { + match_profiler = std::make_unique<vespalib::ExecutionProfiler>(profileDepth); first_phase_profiler = std::make_unique<vespalib::ExecutionProfiler>(profileDepth); second_phase_profiler = std::make_unique<vespalib::ExecutionProfiler>(profileDepth); } @@ -456,6 +464,9 @@ MatchThread::run() trace->addEvent(4, "Start thread merge"); mergeDirector.dualMerge(thread_id, *resultContext->result, resultContext->groupingSource); trace->addEvent(4, "MatchThread::run Done"); + if (match_profiler) { + match_profiler->report(trace->createCursor("match_profiling")); + } if (first_phase_profiler) { first_phase_profiler->report(trace->createCursor("first_phase_profiling"), [](const vespalib::string &name){ return BlueprintResolver::describe_feature(name); }); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index 2a48b26d393..3f97bfe4f08 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -67,6 +67,7 @@ private: double wait_time_s; bool match_with_ranking; std::unique_ptr<Trace> trace; + std::unique_ptr<vespalib::ExecutionProfiler> match_profiler; std::unique_ptr<vespalib::ExecutionProfiler> first_phase_profiler; std::unique_ptr<vespalib::ExecutionProfiler> second_phase_profiler; UniqueIssues my_issues; diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index f02681908d6..b396839b7a2 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -6,13 +6,14 @@ #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/feature.h> #include <vespa/searchlib/fef/matchdata.h> +#include <vespa/searchlib/queryeval/global_filter.h> #include <vespa/searchlib/queryeval/nearest_neighbor_iterator.h> #include <vespa/searchlib/queryeval/nns_index_iterator.h> #include <vespa/searchlib/queryeval/simpleresult.h> -#include <vespa/searchlib/queryeval/global_filter.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/distance_calculator.h> #include <vespa/searchlib/tensor/distance_function_factory.h> +#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/stringfmt.h> @@ -29,6 +30,8 @@ using search::feature_t; using search::tensor::DenseTensorAttribute; using search::tensor::DistanceCalculator; using search::tensor::DistanceFunction; +using search::tensor::SerializedFastValueAttribute; +using search::tensor::TensorAttribute; using vespalib::eval::CellType; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; @@ -38,8 +41,13 @@ using vespalib::eval::ValueType; using namespace search::fef; using namespace search::queryeval; +using BasicType = search::attribute::BasicType; +using CollectionType = search::attribute::CollectionType; +using Config = search::attribute::Config; + vespalib::string denseSpecDouble("tensor(x[2])"); vespalib::string denseSpecFloat("tensor<float>(x[2])"); +vespalib::string mixed_spec("tensor(m{},x[2])"); DistanceFunction::UP euclid_d = search::tensor::make_distance_function(DistanceMetric::Euclidean, CellType::DOUBLE); DistanceFunction::UP euclid_f = search::tensor::make_distance_function(DistanceMetric::Euclidean, CellType::FLOAT); @@ -49,43 +57,45 @@ std::unique_ptr<Value> createTensor(const TensorSpec &spec) { } std::unique_ptr<Value> createTensor(const vespalib::string& type_spec, double v1, double v2) { - return createTensor(TensorSpec(type_spec).add({{"x", 0}}, v1) - .add({{"x", 1}}, v2)); + auto type = vespalib::eval::ValueType::from_spec(type_spec); + if (type.is_dense()) { + return createTensor(TensorSpec(type_spec).add({{"x", 0}}, v1) + .add({{"x", 1}}, v2)); + } else { + return createTensor(TensorSpec(type_spec).add({{"m", "a"},{"x", 0}}, v1) + .add({{"m", "a"},{"x", 1}}, v2)); + } } -struct Fixture -{ - using BasicType = search::attribute::BasicType; - using CollectionType = search::attribute::CollectionType; - using Config = search::attribute::Config; +std::shared_ptr<TensorAttribute> make_attr(const vespalib::string& name, const Config& cfg) { + if (cfg.tensorType().is_dense()) { + return std::make_shared<DenseTensorAttribute>(name, cfg); + } else { + return std::make_shared<SerializedFastValueAttribute>(name, cfg); + } +} +struct Fixture { Config _cfg; vespalib::string _name; vespalib::string _typeSpec; - std::shared_ptr<DenseTensorAttribute> _tensorAttr; - std::shared_ptr<AttributeVector> _attr; + std::shared_ptr<TensorAttribute> _attr; std::shared_ptr<GlobalFilter> _global_filter; Fixture(const vespalib::string &typeSpec) : _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name("test"), _typeSpec(typeSpec), - _tensorAttr(), _attr(), _global_filter(GlobalFilter::create()) { _cfg.setTensorType(ValueType::from_spec(typeSpec)); - _tensorAttr = makeAttr(); - _attr = _tensorAttr; + _attr = make_attr(_name, _cfg); _attr->addReservedDoc(); } ~Fixture() {} - std::shared_ptr<DenseTensorAttribute> makeAttr() { - return std::make_shared<DenseTensorAttribute>(_name, _cfg); - } - void ensureSpace(uint32_t docId) { while (_attr->getNumDocs() <= docId) { uint32_t newDocId = 0u; @@ -101,7 +111,7 @@ struct Fixture void setTensor(uint32_t docId, const Value &tensor) { ensureSpace(docId); - _tensorAttr->setTensor(docId, tensor); + _attr->setTensor(docId, tensor); _attr->commit(); } @@ -123,7 +133,7 @@ template <bool strict> SimpleResult find_matches(Fixture &env, const Value &qtv, double threshold = std::numeric_limits<double>::max()) { auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); - auto &attr = *(env._tensorAttr); + auto &attr = *(env._attr); DistanceCalculator dist_calc(attr, qtv, env.dist_fun()); NearestNeighborDistanceHeap dh(2); dh.set_distance_threshold(env.dist_fun().convert_threshold(threshold)); @@ -172,12 +182,38 @@ verify_iterator_returns_expected_results(const vespalib::string& attribute_tenso EXPECT_EQ(result, far_thr4_exp); result = find_matches<false>(fixture, *farTensor, 4.0); EXPECT_EQ(result, far_thr4_exp); +} +struct TestParam { + vespalib::string attribute_tensor_type_spec; + vespalib::string query_tensor_type_spec; + TestParam(const vespalib::string& attribute_tensor_type_spec_in, + const vespalib::string& query_tensor_type_spec_in) + : attribute_tensor_type_spec(attribute_tensor_type_spec_in), + query_tensor_type_spec(query_tensor_type_spec_in) + {} + ~TestParam() {} +}; + +std::ostream& operator<<(std::ostream& os, const TestParam& param) +{ + os << "{" << param.attribute_tensor_type_spec << ", " << param.query_tensor_type_spec << "}"; + return os; } -TEST(NnsIndexIteratorTest, require_that_iterator_returns_expected_results) { - verify_iterator_returns_expected_results(denseSpecDouble, denseSpecDouble); - verify_iterator_returns_expected_results(denseSpecFloat, denseSpecFloat); +struct NnsIndexIteratorParameterizedTest : public ::testing::TestWithParam<TestParam> {}; + +INSTANTIATE_TEST_SUITE_P(NnsTestSuite, + NnsIndexIteratorParameterizedTest, + ::testing::Values( + TestParam(denseSpecDouble, denseSpecDouble), + TestParam(denseSpecFloat, denseSpecFloat), + TestParam(mixed_spec, denseSpecDouble) + )); + +TEST_P(NnsIndexIteratorParameterizedTest, require_that_iterator_returns_expected_results) { + auto param = GetParam(); + verify_iterator_returns_expected_results(param.attribute_tensor_type_spec, param.query_tensor_type_spec); } void @@ -207,16 +243,16 @@ verify_iterator_returns_filtered_results(const vespalib::string& attribute_tenso EXPECT_EQ(result, farExpect); } -TEST(NnsIndexIteratorTest, require_that_iterator_returns_filtered_results) { - verify_iterator_returns_filtered_results(denseSpecDouble, denseSpecDouble); - verify_iterator_returns_filtered_results(denseSpecFloat, denseSpecFloat); +TEST_P(NnsIndexIteratorParameterizedTest, require_that_iterator_returns_filtered_results) { + auto param = GetParam(); + verify_iterator_returns_filtered_results(param.attribute_tensor_type_spec, param.query_tensor_type_spec); } template <bool strict> std::vector<feature_t> get_rawscores(Fixture &env, const Value &qtv) { auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); - auto &attr = *(env._tensorAttr); + auto &attr = *(env._attr); DistanceCalculator dist_calc(attr, qtv, env.dist_fun()); NearestNeighborDistanceHeap dh(2); auto dummy_filter = GlobalFilter::create(); @@ -261,9 +297,29 @@ verify_iterator_sets_expected_rawscore(const vespalib::string& attribute_tensor_ } } -TEST(NnsIndexIteratorTest, require_that_iterator_sets_expected_rawscore) { - verify_iterator_sets_expected_rawscore(denseSpecDouble, denseSpecDouble); - verify_iterator_sets_expected_rawscore(denseSpecFloat, denseSpecFloat); +TEST_P(NnsIndexIteratorParameterizedTest, require_that_iterator_sets_expected_rawscore) { + auto param = GetParam(); + verify_iterator_sets_expected_rawscore(param.attribute_tensor_type_spec, param.query_tensor_type_spec); +} + +void expect_match(SearchIterator& itr, uint32_t docid) { + bool match = itr.seek(docid); + EXPECT_TRUE(match); + EXPECT_FALSE(itr.isAtEnd()); + EXPECT_EQ(docid, itr.getDocId()); +} + +void expect_not_match(SearchIterator& itr, uint32_t curr_docid, uint32_t exp_next_docid) { + bool match = itr.seek(curr_docid); + EXPECT_FALSE(match); + EXPECT_FALSE(itr.isAtEnd()); + EXPECT_EQ(exp_next_docid, itr.getDocId()); +} + +void expect_at_end(SearchIterator& itr, uint32_t docid) { + bool match = itr.seek(docid); + EXPECT_FALSE(match); + EXPECT_TRUE(itr.isAtEnd()); } TEST(NnsIndexIteratorTest, require_that_iterator_works_as_expected) { @@ -271,65 +327,29 @@ TEST(NnsIndexIteratorTest, require_that_iterator_works_as_expected) { auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); auto search = NnsIndexIterator::create(tfmd, hits, *euclid_d); - uint32_t docid = 1; search->initFullRange(); - bool match = search->seek(docid); - EXPECT_FALSE(match); - EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQ(2u, search->getDocId()); - docid = 2; - match = search->seek(docid); - EXPECT_TRUE(match); - EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQ(docid, search->getDocId()); - search->unpack(docid); + expect_not_match(*search, 1, 2); + expect_match(*search, 2); + search->unpack(2); EXPECT_NEAR(1.0/(1.0+2.0), tfmd.getRawScore(), EPS); - docid = 3; - match = search->seek(docid); - EXPECT_TRUE(match); - EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQ(docid, search->getDocId()); - search->unpack(docid); + expect_match(*search, 3); + search->unpack(3); EXPECT_NEAR(1.0/(1.0+3.0), tfmd.getRawScore(), EPS); - docid = 4; - match = search->seek(docid); - EXPECT_FALSE(match); - EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQ(5u, search->getDocId()); - - docid = 6; - match = search->seek(docid); - EXPECT_FALSE(match); - EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQ(8u, search->getDocId()); - docid = 8; - search->unpack(docid); + expect_not_match(*search, 4, 5); + expect_not_match(*search, 6, 8); + search->unpack(8); EXPECT_NEAR(1.0/(1.0+4.0), tfmd.getRawScore(), EPS); - docid = 9; - match = search->seek(docid); - EXPECT_TRUE(match); - EXPECT_FALSE(search->isAtEnd()); - docid = 10; - match = search->seek(docid); - EXPECT_FALSE(match); - EXPECT_TRUE(search->isAtEnd()); - docid = 4; - search->initRange(docid, 7); - match = search->seek(docid); - EXPECT_FALSE(match); - EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQ(5u, search->getDocId()); - docid = 5; - search->unpack(docid); + expect_match(*search, 9); + expect_at_end(*search, 10); + + search->initRange(4, 7); + expect_not_match(*search, 4, 5); + search->unpack(5); EXPECT_NEAR(1.0/(1.0+1.0), tfmd.getRawScore(), EPS); - EXPECT_FALSE(search->isAtEnd()); - docid = 6; - match = search->seek(docid); - EXPECT_FALSE(match); - EXPECT_TRUE(search->isAtEnd()); + expect_at_end(*search, 6); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 03759e8a5cc..d22f24cc7da 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -451,8 +451,7 @@ template <HnswIndexType type> void HnswIndex<type>::internal_prepare_add_node(typename HnswIndex::PreparedAddDoc& op, TypedCells input_vector, const typename GraphType::EntryNode& entry) const { - // TODO: Add capping on num_levels - int node_max_level = _level_generator->max_level(); + int node_max_level = std::min(_level_generator->max_level(), max_max_level); std::vector<typename PreparedAddNode::Links> connections(node_max_level + 1); if (entry.nodeid == 0) { // graph has no entry point diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 272c3df5f2f..617eb85076e 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -75,6 +75,10 @@ protected: } } + // Clamp level generator member function max_level() return value + // Chosen value is based on class comment for InvLogLevelGenerator. + static constexpr uint32_t max_max_level = 29; + GraphType _graph; const DocVectorAccess& _vectors; DistanceFunction::UP _distance_func; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index b63034acfc4..9ee8d9fdf46 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -83,9 +83,10 @@ uint32_t TensorAttribute::clearDoc(DocId docId) { consider_remove_from_index(docId); - EntryRef oldRef(_refVector[docId].load_relaxed()); updateUncommittedDocIdLimit(docId); - _refVector[docId] = AtomicEntryRef(); + auto& elem_ref = _refVector[docId]; + EntryRef oldRef(elem_ref.load_relaxed()); + elem_ref.store_relaxed(EntryRef()); if (oldRef.valid()) { _tensorStore.holdTensor(oldRef); return 1u; @@ -96,7 +97,6 @@ TensorAttribute::clearDoc(DocId docId) void TensorAttribute::onCommit() { - // Note: Cost can be reduced if unneeded generation increments are dropped incGeneration(); if (_tensorStore.consider_compact()) { auto context = _tensorStore.start_compact(getConfig().getCompactionStrategy()); @@ -178,11 +178,9 @@ TensorAttribute::setTensorRef(DocId docId, EntryRef ref) { assert(docId < _refVector.size()); updateUncommittedDocIdLimit(docId); - // TODO: validate if following fence is sufficient. - std::atomic_thread_fence(std::memory_order_release); - // TODO: Check if refVector must consist of std::atomic<EntryRef> - EntryRef oldRef(_refVector[docId].load_relaxed()); - _refVector[docId].store_release(ref); + auto& elem_ref = _refVector[docId]; + EntryRef oldRef(elem_ref.load_relaxed()); + elem_ref.store_release(ref); if (oldRef.valid()) { _tensorStore.holdTensor(oldRef); } diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index 702192a25ab..6b1faadfb1d 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -163,7 +163,7 @@ export MALLOC_ARENA_MAX=1 optionally_reduce_base_frequency # Prefer newer gdb and pstack -prepend_path /opt/rh/gcc-toolset-11/root/usr/bin +prepend_path /opt/rh/gcc-toolset-12/root/usr/bin # Maven is needed for tester applications prepend_path "$VESPA_HOME/local/maven/bin" |