diff options
83 files changed, 776 insertions, 446 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java index 34e86cbf4a8..70fcf64dff3 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java @@ -100,7 +100,8 @@ public final class Attribute implements Cloneable, Serializable { BOOL("bool", "BOOL"), PREDICATE("predicate", "PREDICATE"), TENSOR("tensor", "TENSOR"), - REFERENCE("reference", "REFERENCE"); + REFERENCE("reference", "REFERENCE"), + RAW("raw", "RAW"); private final String myName; // different from what name() returns. private final String exportAttributeTypeName; @@ -290,7 +291,7 @@ public final class Attribute implements Cloneable, Serializable { } else if (fval instanceof ByteFieldValue) { return Type.BYTE; } else if (fval instanceof Raw) { - return Type.BYTE; + return Type.RAW; } else if (fval instanceof PredicateFieldValue) { return Type.PREDICATE; } else if (fval instanceof TensorFieldValue) { @@ -344,6 +345,7 @@ public final class Attribute implements Cloneable, Serializable { case PREDICATE -> DataType.PREDICATE; case TENSOR -> DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new)); case REFERENCE-> createReferenceDataType(); + case RAW -> DataType.RAW; default -> throw new IllegalArgumentException("Unknown attribute type " + attributeType); }; } diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index 6e7787c2dd1..993a2442fdb 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -13,6 +13,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; import com.yahoo.tensor.functions.Generate; +import com.yahoo.tensor.functions.Slice; import java.io.StringReader; import java.util.HashSet; @@ -53,6 +54,13 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { } return transformChildren(t, childContext); } + if (f instanceof Slice s) { + for (var tf : s.selectorFunctions()) { + if (tf instanceof TensorFunctionNode.ExpressionTensorFunction expr) { + transform(expr.wrappedExpression(), context); + } + } + } } if (node instanceof CompositeNode c) return transformChildren(c, context); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index 8f7e8daeed0..df4b0d0d941 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -89,7 +89,7 @@ public class Processing { OnnxModelConfigGenerator::new, OnnxModelTypeResolver::new, RankingExpressionTypeResolver::new, - BoolAttributeValidator::new, + SingleValueOnlyAttributeValidator::new, PagedAttributeValidator::new, // These should be last: IndexingValidation::new, diff --git a/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidator.java index bdb1eed4b10..b2786e6c785 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidator.java @@ -14,9 +14,9 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * * @author geirst */ -public class BoolAttributeValidator extends Processor { +public class SingleValueOnlyAttributeValidator extends Processor { - public BoolAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + public SingleValueOnlyAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } @@ -27,9 +27,10 @@ public class BoolAttributeValidator extends Processor { if (attribute == null) { continue; } - if (attribute.getType().equals(Attribute.Type.BOOL) && + if ((attribute.getType().equals(Attribute.Type.BOOL) || + attribute.getType().equals(Attribute.Type.RAW)) && !attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE)) { - fail(schema, field, "Only single value bool attribute fields are supported"); + fail(schema, field, "Only single value " + attribute.getType().getName() + " attribute fields are supported"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java index da4cabcf9f5..35d7b2d225e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java @@ -95,7 +95,8 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr serviceBuilder.environ(b -> b.varname(entry.getKey()).varvalue(entry.getValue().toString())); } for (var entry : s.getLogctlSpecs()) { - serviceBuilder.logctl(b -> b.componentSpec(entry.componentSpec).levelsModSpec(entry.levelsModSpec)); + serviceBuilder.logctl(b -> b.componentSpec(entry.componentSpec()) + .levelsModSpec(entry.levelsModSpec().toLogctlModSpec())); } setPreShutdownCommand(serviceBuilder, s); return serviceBuilder; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java index fd959943eab..d927c38082e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java @@ -1,11 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; -public class LogctlSpec { - public final String componentSpec; - public final String levelsModSpec; - public LogctlSpec(String componentSpec, String levelsModSpec) { - this.componentSpec = componentSpec; - this.levelsModSpec = levelsModSpec; - } +import com.yahoo.container.logging.LevelsModSpec; + +public record LogctlSpec(String componentSpec, LevelsModSpec levelsModSpec) { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 317c3646dde..62cbf61b336 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.container.logging.LevelsModSpec; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.ConfigProxy; import com.yahoo.vespa.model.ConfigSentinel; @@ -24,7 +25,6 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; - import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; @@ -41,8 +41,6 @@ import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty; */ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Serializable { - private static final long serialVersionUID = 1L; - private final boolean isHostedVespa; private final Monitoring monitoring; private final List<Configserver> configservers = new ArrayList<>(); @@ -68,12 +66,6 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri } private final List<LogctlSpec> logctlSpecs = new ArrayList<>(); - public List<LogctlSpec> getLogctlSpecs() { - return logctlSpecs; - } - public void addLogctlCommand(String componentSpec, String levelsModSpec) { - logctlSpecs.add(new LogctlSpec(componentSpec, levelsModSpec)); - } /** * The single cluster controller cluster shared by all content clusters by default when not multitenant. @@ -101,6 +93,7 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri this.multitenant = multitenant; this.fileDistribution = new FileDistributionConfigProducer(parent); this.applicationType = applicationType; + this.logctlSpecs.addAll(defaultLogctlSpecs()); } public Configserver getConfigserver() { return defaultConfigserver; } @@ -331,4 +324,25 @@ public class Admin extends TreeConfigProducer<AnyConfigProducer> implements Seri public ApplicationType getApplicationType() { return applicationType; } + public List<LogctlSpec> getLogctlSpecs() { + return logctlSpecs; + } + public void addLogctlCommand(String componentSpec, LevelsModSpec levelsModSpec) { + logctlSpecs.add(new LogctlSpec(componentSpec, levelsModSpec)); + } + + private static List<LogctlSpec> defaultLogctlSpecs() { + // Turn off info logging for all containers for some classes (unimportant log messages that create noise in vespa log) + return List.of(new LogctlSpec("com.yahoo.vespa.spifly.repackaged.spifly.BaseActivator", getLevelModSpec("-info")), + new LogctlSpec("org.eclipse.jetty.server.Server", getLevelModSpec("-info")), + new LogctlSpec("org.eclipse.jetty.server.handler.ContextHandler", getLevelModSpec("-info")), + new LogctlSpec("org.eclipse.jetty.server.AbstractConnector", getLevelModSpec("-info"))); + } + + static LevelsModSpec getLevelModSpec(String levels) { + var levelSpec = new LevelsModSpec(); + levelSpec.setLevels(levels); + return levelSpec; + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index e50c010c476..fef7d534c30 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -25,9 +25,6 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain } @Override - protected void doPrepare(DeployState deployState) { } - - @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); builder.jvm.heapsize(128); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 25d5620eff9..c65abfb0189 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -37,9 +37,6 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC @Override protected Set<Path> unnecessaryPlatformBundles() { return UNNECESSARY_BUNDLES; } - @Override - protected void doPrepare(DeployState deployState) { } - @Override protected boolean messageBusEnabled() { return false; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index ec543dba6fb..ada647b535d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -137,9 +137,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } @Override - protected void doPrepare(DeployState deployState) { } - - @Override public void getConfig(MetricsNodesConfig.Builder builder) { builder.node.addAll(MetricsNodesConfigGenerator.generate(getContainers())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java index 34ec5de975d..1977a584713 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java @@ -4,11 +4,20 @@ package com.yahoo.vespa.model.admin.monitoring; import com.yahoo.metrics.ContainerMetrics; import com.yahoo.metrics.SearchNodeMetrics; +import com.yahoo.metrics.Suffix; +import com.yahoo.metrics.VespaMetrics; import java.util.Collections; +import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.Set; +import static com.yahoo.metrics.Suffix.average; +import static com.yahoo.metrics.Suffix.count; +import static com.yahoo.metrics.Suffix.max; +import static com.yahoo.metrics.Suffix.ninety_five_percentile; +import static com.yahoo.metrics.Suffix.ninety_nine_percentile; +import static com.yahoo.metrics.Suffix.sum; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; /** @@ -39,72 +48,53 @@ public class DefaultMetrics { } private static void addContainerMetrics(Set<Metric> metrics) { - metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_1XX.rate())); - metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_2XX.rate())); - metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_3XX.rate())); - metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_4XX.rate())); - metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_5XX.rate())); - metrics.add(new Metric(ContainerMetrics.JDISC_GC_MS.average())); - metrics.add(new Metric(ContainerMetrics.MEM_HEAP_FREE.average())); + addMetric(metrics, ContainerMetrics.HTTP_STATUS_1XX.rate()); + addMetric(metrics, ContainerMetrics.HTTP_STATUS_2XX.rate()); + addMetric(metrics, ContainerMetrics.HTTP_STATUS_3XX.rate()); + addMetric(metrics, ContainerMetrics.HTTP_STATUS_4XX.rate()); + addMetric(metrics, ContainerMetrics.HTTP_STATUS_5XX.rate()); + addMetric(metrics, ContainerMetrics.JDISC_GC_MS.average()); + addMetric(metrics, ContainerMetrics.MEM_HEAP_FREE.average()); } private static void addSearchChainMetrics(Set<Metric> metrics) { - metrics.add(new Metric(ContainerMetrics.QUERIES.rate())); - metrics.add(new Metric(ContainerMetrics.QUERY_LATENCY.sum())); - metrics.add(new Metric(ContainerMetrics.QUERY_LATENCY.count())); - metrics.add(new Metric(ContainerMetrics.QUERY_LATENCY.max())); - metrics.add(new Metric(ContainerMetrics.QUERY_LATENCY.ninety_five_percentile())); - metrics.add(new Metric(ContainerMetrics.QUERY_LATENCY.ninety_nine_percentile())); - metrics.add(new Metric(ContainerMetrics.HITS_PER_QUERY.sum())); - metrics.add(new Metric(ContainerMetrics.HITS_PER_QUERY.count())); - metrics.add(new Metric(ContainerMetrics.HITS_PER_QUERY.max())); - metrics.add(new Metric(ContainerMetrics.TOTAL_HITS_PER_QUERY.sum())); - metrics.add(new Metric(ContainerMetrics.TOTAL_HITS_PER_QUERY.count())); - metrics.add(new Metric(ContainerMetrics.TOTAL_HITS_PER_QUERY.max())); - metrics.add(new Metric(ContainerMetrics.DEGRADED_QUERIES.rate())); - metrics.add(new Metric(ContainerMetrics.FAILED_QUERIES.rate())); - metrics.add(new Metric(ContainerMetrics.QUERY_LATENCY.average())); // TODO: Remove with Vespa 9 - metrics.add(new Metric(ContainerMetrics.HITS_PER_QUERY.average())); // TODO: Remove with Vespa 9 - metrics.add(new Metric(ContainerMetrics.TOTAL_HITS_PER_QUERY.average())); // TODO: Remove with Vespa 9 - metrics.add(new Metric(ContainerMetrics.SERVER_ACTIVE_THREADS.average())); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + addMetric(metrics, ContainerMetrics.QUERIES.rate()); + addMetric(metrics, ContainerMetrics.QUERY_LATENCY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile, average)); // TODO: Remove average with Vespa 9 + addMetric(metrics, ContainerMetrics.HITS_PER_QUERY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 + addMetric(metrics, ContainerMetrics.TOTAL_HITS_PER_QUERY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 + addMetric(metrics, ContainerMetrics.DEGRADED_QUERIES.rate()); + addMetric(metrics, ContainerMetrics.FAILED_QUERIES.rate()); + addMetric(metrics, ContainerMetrics.SERVER_ACTIVE_THREADS.average()); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. } private static void addContentMetrics(Set<Metric> metrics) { - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_REQUESTED_DOCUMENTS.rate())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_LATENCY.sum())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_LATENCY.count())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_LATENCY.max())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_LATENCY.average())); // TODO: Remove with Vespa 9 - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_LATENCY.sum())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_LATENCY.count())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_LATENCY.max())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_LATENCY.average())); // TODO: Remove with Vespa 9 - - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DISK_USAGE.last())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ALLOCATED_BYTES.last())); - - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average())); - - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_MATCHED.rate())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_RERANKED.rate())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_SETUP_TIME.sum())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_SETUP_TIME.count())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_SETUP_TIME.max())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_SETUP_TIME.average())); // TODO: Remove with Vespa 9 - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_LATENCY.sum())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_LATENCY.count())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_LATENCY.max())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_LATENCY.average())); // TODO: Remove with Vespa 9 - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME.sum())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME.count())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME.max())); - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME.average())); // TODO: Remove with Vespa 9 - - metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_DISK_USAGE.last())); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_REQUESTED_DOCUMENTS.rate()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_LATENCY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_LATENCY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 + + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DISK_USAGE.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ALLOCATED_BYTES.last()); + + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); + + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_MATCHED.rate()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_DOCS_RERANKED.rate()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_SETUP_TIME, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_QUERY_LATENCY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_RANK_PROFILE_RERANK_TIME, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_DISK_USAGE.last()); + } + + private static void addMetric(Set<Metric> metrics, String nameWithSuffix) { + metrics.add(new Metric(nameWithSuffix)); + } + + private static void addMetric(Set<Metric> metrics, VespaMetrics metric, EnumSet<Suffix> suffixes) { + suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); } private DefaultMetrics() { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 5f8d8148b41..9c0a6b1b224 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import com.yahoo.metrics.ClusterControllerMetrics; import com.yahoo.metrics.ConfigServerMetrics; import com.yahoo.metrics.ContainerMetrics; import com.yahoo.metrics.DistributorMetrics; @@ -12,6 +13,7 @@ import com.yahoo.metrics.SlobrokMetrics; import com.yahoo.metrics.StorageMetrics; import com.yahoo.metrics.NodeAdminMetrics; import com.yahoo.metrics.Suffix; +import com.yahoo.metrics.VespaMetrics; import java.util.Collections; import java.util.EnumSet; @@ -83,7 +85,7 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_SERVER_TLS_CONNECIONTS_ESTABLISHED.baseName()); addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_CLIENT_TLS_CONNECTIONS_ESTABLISHED.baseName()); addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_SERVER_UNENCRYPTED_CONNECTIONS_ESTABLISHED.baseName()); - addMetric(metrics, ContainerMetrics. JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED. baseName()); + addMetric(metrics, ContainerMetrics.JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED.baseName()); // C++ TLS metrics addMetric(metrics, StorageMetrics.VDS_SERVER_NETWORK_TLS_HANDSHAKES_FAILED.count()); @@ -112,7 +114,7 @@ public class VespaMetricSet { } private static Set<Metric> getConfigServerMetrics() { - Set<Metric> metrics =new LinkedHashSet<>(); + Set<Metric> metrics = new LinkedHashSet<>(); addMetric(metrics, ConfigServerMetrics.REQUESTS.count()); addMetric(metrics, ConfigServerMetrics.FAILED_REQUESTS.count()); @@ -139,9 +141,9 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.HANDLED_REQUESTS.count()); addMetric(metrics, ContainerMetrics.HANDLED_LATENCY, EnumSet.of(sum, count, max)); - - addMetric(metrics, ContainerMetrics.SERVER_NUM_OPEN_CONNECTIONS, EnumSet.of(max,last, average)); - addMetric(metrics, ContainerMetrics.SERVER_NUM_CONNECTIONS, EnumSet.of(max,last, average)); + + addMetric(metrics, ContainerMetrics.SERVER_NUM_OPEN_CONNECTIONS, EnumSet.of(max, last, average)); + addMetric(metrics, ContainerMetrics.SERVER_NUM_CONNECTIONS, EnumSet.of(max, last, average)); addMetric(metrics, ContainerMetrics.SERVER_BYTES_RECEIVED, EnumSet.of(sum, count)); addMetric(metrics, ContainerMetrics.SERVER_BYTES_SENT, EnumSet.of(sum, count)); @@ -186,7 +188,7 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.MEM_NATIVE_TOTAL.average()); addMetric(metrics, ContainerMetrics.MEM_NATIVE_FREE.average()); addMetric(metrics, ContainerMetrics.MEM_NATIVE_USED.average()); - + addMetric(metrics, ContainerMetrics.JDISC_MEMORY_MAPPINGS.max()); addMetric(metrics, ContainerMetrics.JDISC_OPEN_FILE_DESCRIPTORS.max()); @@ -239,7 +241,7 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.JDISC_APPLICATION_FAILED_COMPONENT_GRAPHS.rate()); addMetric(metrics, ContainerMetrics.JDISC_JVM.last()); - + // Deprecated metrics. TODO: Remove on Vespa 9. addMetric(metrics, ContainerMetrics.SERVER_REJECTED_REQUESTS, EnumSet.of(rate, count)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. addMetric(metrics, ContainerMetrics.SERVER_THREAD_POOL_SIZE, EnumSet.of(max, last)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. @@ -254,32 +256,30 @@ public class VespaMetricSet { private static Set<Metric> getClusterControllerMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_DOWN_COUNT.last()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_INITIALIZING_COUNT.last()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_MAINTENANCE_COUNT.last()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_RETIRED_COUNT.last()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_STOPPING_COUNT.last()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_UP_COUNT.last()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_CLUSTER_STATE_CHANGE_COUNT.baseName()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_BUSY_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_IDLE_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); - - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_WORK_MS, EnumSet.of(last, sum, count)); - - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_IS_MASTER.last()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_REMOTE_TASK_QUEUE_SIZE.last()); + addMetric(metrics, ClusterControllerMetrics.DOWN_COUNT.last()); + addMetric(metrics, ClusterControllerMetrics.INITIALIZING_COUNT.last()); + addMetric(metrics, ClusterControllerMetrics.MAINTENANCE_COUNT.last()); + addMetric(metrics, ClusterControllerMetrics.RETIRED_COUNT.last()); + addMetric(metrics, ClusterControllerMetrics.STOPPING_COUNT.last()); + addMetric(metrics, ClusterControllerMetrics.UP_COUNT.last()); + addMetric(metrics, ClusterControllerMetrics.CLUSTER_STATE_CHANGE_COUNT.baseName()); + addMetric(metrics, ClusterControllerMetrics.BUSY_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); + addMetric(metrics, ClusterControllerMetrics.IDLE_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); + + addMetric(metrics, ClusterControllerMetrics.WORK_MS, EnumSet.of(last, sum, count)); + + addMetric(metrics, ClusterControllerMetrics.IS_MASTER.last()); + addMetric(metrics, ClusterControllerMetrics.REMOTE_TASK_QUEUE_SIZE.last()); // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. // DO NOT RELY ON THIS METRIC YET. - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_NODE_EVENT_COUNT.baseName()); - - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(last, max)); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max)); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max)); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MEMORY_LIMIT.last()); - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_DISK_LIMIT.last()); - - addMetric(metrics, ContainerMetrics.CLUSTER_CONTROLLER_REINDEXING_PROGRESS.last()); - + addMetric(metrics, ClusterControllerMetrics.NODE_EVENT_COUNT.baseName()); + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(last, max)); + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max)); + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max)); + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MEMORY_LIMIT.last()); + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_DISK_LIMIT.last()); + addMetric(metrics, ClusterControllerMetrics.REINDEXING_PROGRESS.last()); + return metrics; } @@ -322,7 +322,7 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.RELEVANCE_AT_1, EnumSet.of(sum, count)); addMetric(metrics, ContainerMetrics.RELEVANCE_AT_3, EnumSet.of(sum, count)); addMetric(metrics, ContainerMetrics.RELEVANCE_AT_10, EnumSet.of(sum, count)); - + // Errors from search container addMetric(metrics, ContainerMetrics.ERROR_TIMEOUT.rate()); addMetric(metrics, ContainerMetrics.ERROR_BACKENDS_OOS.rate()); @@ -420,7 +420,7 @@ public class VespaMetricSet { addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_ACCEPTED.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_UTILIZATION, EnumSet.of(max, sum, count)); - + // lid space addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); @@ -617,6 +617,7 @@ public class VespaMetricSet { return metrics; } + private static Set<Metric> getDistributorMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS_RECHECKING.average()); @@ -710,33 +711,10 @@ public class VespaMetricSet { metrics.add(new Metric(nameWithSuffix)); } - private static void addMetric(Set<Metric> metrics, ContainerMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } - - private static void addMetric(Set<Metric> metrics, SearchNodeMetrics metric, EnumSet<Suffix> suffixes) { + private static void addMetric(Set<Metric> metrics, VespaMetrics metric, EnumSet<Suffix> suffixes) { suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); } - private static void addMetric(Set<Metric> metrics, StorageMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } - - private static void addMetric(Set<Metric> metrics, DistributorMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } - private static void addMetric(Set<Metric> metrics, SentinelMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } - private static void addMetric(Set<Metric> metrics, SlobrokMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } - private static void addMetric(Set<Metric> metrics, LogdMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } - private static void addMetric(Set<Metric> metrics, ConfigServerMetrics metric, EnumSet<Suffix> suffixes) { - suffixes.forEach(suffix -> metrics.add(new Metric(metric.baseName() + "." + suffix.suffix()))); - } private static void addMetric(Set<Metric> metrics, String metricName, Iterable<String> aggregateSuffices) { for (String suffix : aggregateSuffices) { metrics.add(new Metric(metricName + "." + suffix)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 57113ff794f..9280f0ceb9a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -21,11 +21,8 @@ import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets; import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder; import org.w3c.dom.Element; - import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; /** @@ -120,7 +117,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu String levels = loggingSpec.requiredStringAttribute("levels"); var levelSpec = new LevelsModSpec(); levelSpec.setLevels(levels); - admin.addLogctlCommand(componentSpec, levelSpec.toLogctlModSpec()); + admin.addLogctlCommand(componentSpec, levelSpec); } void addLoggingSpecs(ModelElement loggingElement, Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index bc410670d5e..f901bf3c826 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -9,11 +9,8 @@ import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.search.config.QrStartConfig; -import com.yahoo.vespa.model.LogctlSpec; import com.yahoo.vespa.model.container.component.SimpleComponent; - import java.time.Duration; -import java.util.List; import java.util.Optional; /** @@ -43,15 +40,6 @@ public final class ApplicationContainer extends Container implements addComponent(new SimpleComponent("com.yahoo.container.jdisc.ClusterInfoProvider")); } - private List<LogctlSpec> logctlSpecs = List.of(); - void setLogctlSpecs(List<LogctlSpec> logctlSpecs) { - this.logctlSpecs = logctlSpecs; - } - @Override - public List<LogctlSpec> getLogctlSpecs() { - return logctlSpecs; - } - @Override public void getConfig(QrStartConfig.Builder builder) { if (getHostResource() != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 004ff5516ba..d101cea428e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -124,19 +124,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat : defaultHeapSizePercentageOfTotalNodeMemory; } - private void wireLogctlSpecs() { - getAdmin().ifPresent(admin -> { - for (var c : getContainers()) { - c.setLogctlSpecs(admin.getLogctlSpecs()); - }}); - } - @Override protected void doPrepare(DeployState deployState) { + super.doPrepare(deployState); addAndSendApplicationBundles(deployState); sendUserConfiguredFiles(deployState); createEndpointList(deployState); - wireLogctlSpecs(); } private void addAndSendApplicationBundles(DeployState deployState) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index be9f3fa894f..2ca6d5d7155 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -16,6 +16,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.LogctlSpec; import com.yahoo.vespa.model.PortAllocBridge; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.container.component.Component; @@ -34,6 +35,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -65,6 +67,8 @@ public abstract class Container extends AbstractService implements /** The cluster this container belongs to, or null if it is not added to any cluster */ private ContainerCluster<?> owner = null; + private List<LogctlSpec> logctlSpecs = List.of(); + protected final TreeConfigProducer<?> parent; private final String name; private boolean requireSpecificPorts = true; @@ -415,4 +419,11 @@ public abstract class Container extends AbstractService implements return producer instanceof ContainerCluster<?> ? (ContainerCluster<?>) producer : null; } + void setLogctlSpecs(List<LogctlSpec> logctlSpecs) { + this.logctlSpecs = logctlSpecs; + } + + @Override + public List<LogctlSpec> getLogctlSpecs() { return logctlSpecs; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index d786135d3ee..45060f2f27d 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -61,7 +61,6 @@ import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.search.SearchCluster; - import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -301,7 +300,16 @@ public abstract class ContainerCluster<CONTAINER extends Container> doPrepare(deployState); } - protected abstract void doPrepare(DeployState deployState); + protected void doPrepare(DeployState deployState) { + wireLogctlSpecs(); + } + + private void wireLogctlSpecs() { + getAdmin().ifPresent(admin -> { + for (var c : getContainers()) { + c.setLogctlSpecs(admin.getLogctlSpecs()); + }}); + } public String getName() { return name; diff --git a/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg b/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg index 33381060178..bf47dba9a71 100644 --- a/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg +++ b/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg @@ -32,9 +32,11 @@ rankprofile[].fef.property[].value "tensor<float>(d0[1],d1[128])" rankprofile[].fef.property[].name "vespa.rank.globalphase" rankprofile[].fef.property[].value "rankingExpression(globalphase)" rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" -rankprofile[].fef.property[].value "onnx(my_ranking_model).score{d0:0}" +rankprofile[].fef.property[].value "onnx(my_ranking_model).score{d0:(attribute(outputidx))}" rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "attribute(tokens)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(outputidx)" rankprofile[].fef.property[].name "vespa.globalphase.rerankcount" rankprofile[].fef.property[].value "1000" rankprofile[].fef.property[].name "vespa.type.attribute.tokens" diff --git a/config-model/src/test/derived/globalphase_token_functions/test.sd b/config-model/src/test/derived/globalphase_token_functions/test.sd index f8cc8863ad1..a1d14258aab 100644 --- a/config-model/src/test/derived/globalphase_token_functions/test.sd +++ b/config-model/src/test/derived/globalphase_token_functions/test.sd @@ -8,6 +8,9 @@ schema test { field tokens type tensor(d0[128]) { indexing: attribute } + field outputidx type double { + indexing: attribute + } } fieldset default { fields: title @@ -35,7 +38,7 @@ schema test { } global-phase { rerank-count: 1000 - expression: onnx(my_ranking_model).score{d0:0} + expression: onnx(my_ranking_model).score{d0:attribute(outputidx)} } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java deleted file mode 100644 index f19b1f43115..00000000000 --- a/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.processing; - -import com.yahoo.schema.parser.ParseException; -import org.junit.jupiter.api.Test; - -import static com.yahoo.schema.ApplicationBuilder.createFromString; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author geirst - */ -public class BoolAttributeValidatorTestCase { - - @Test - void array_of_bool_attribute_is_not_supported() throws ParseException { - try { - createFromString(getSd("field b type array<bool> { indexing: attribute }")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported", - e.getMessage()); - } - } - - @Test - void weigtedset_of_bool_attribute_is_not_supported() throws ParseException { - try { - createFromString(getSd("field b type weightedset<bool> { indexing: attribute }")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported", - e.getMessage()); - } - } - - private String getSd(String field) { - return joinLines( - "schema test {", - " document test {", - " " + field, - " }", - "}"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidatorTestCase.java new file mode 100644 index 00000000000..a7f4125a537 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidatorTestCase.java @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; + +import static com.yahoo.schema.ApplicationBuilder.createFromString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author geirst + */ +public class SingleValueOnlyAttributeValidatorTestCase { + + private static void array_attribute_is_not_supported(String type) throws ParseException { + try { + createFromString(getSd("field b type array<" + type + "> { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'b': Only single value " + type + " attribute fields are supported", + e.getMessage()); + } + } + + private static void weightedset_attribute_is_not_supported(String type) throws ParseException { + try { + createFromString(getSd("field b type weightedset<" + type + "> { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + if (type.equals("raw")) { + assertEquals("weightedset of complex type '[type BUILTIN] {raw}' is not supported", + e.getMessage()); + } else { + assertEquals("For schema 'test', field 'b': Only single value " + type + " attribute fields are supported", + e.getMessage()); + } + } + } + + @Test + void array_of_bool_attribute_is_not_supported() throws ParseException { + array_attribute_is_not_supported("bool"); + } + + @Test + void weightedset_of_bool_attribute_is_not_supported() throws ParseException { + weightedset_attribute_is_not_supported("bool"); + } + + @Test + void array_of_raw_attribute_is_not_supported() throws ParseException { + array_attribute_is_not_supported("raw"); + } + + @Test + void weightedset_of_raw_attribute_is_not_supported() throws ParseException { + weightedset_attribute_is_not_supported("raw"); + } + + private static String getSd(String field) { + return joinLines( + "schema test {", + " document test {", + " " + field, + " }", + "}"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index d060fdfd3d2..5559eb633ef 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -17,17 +17,22 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.net.HostName; import com.yahoo.vespa.config.core.StateserverConfig; +import com.yahoo.vespa.model.LogctlSpec; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Test; - +import java.util.List; import java.util.Set; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertTrue; public class AdminTestCase { @@ -243,4 +248,49 @@ public class AdminTestCase { assertTrue(configIds.contains("hosts/myhost0/logforwarder"), configIds.toString()); } + @Test + void testDefaultLogCtlSpecs() { + String hosts = "<hosts>" + + " <host name=\"myhost0\">" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + + String services = "<services>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " </admin>" + + " <container version=\"1.0\">" + + " <nodes>" + + " <node hostalias=\"node0\" />" + + " </nodes>" + + " <search/>" + + " <document-api/>" + + " </container>" + + "</services>"; + + VespaModel vespaModel = new VespaModelCreatorWithMockPkg(hosts, services).create(); + List<LogctlSpec> logctlSpecs = vespaModel.getAdmin().getLogctlSpecs(); + assertEquals(4, logctlSpecs.size()); // Default logctl specs + assertEquals(1, logctlSpecs + .stream() + .filter(l -> (l.componentSpec() + .equals("com.yahoo.vespa.spifly.repackaged.spifly.BaseActivator") + && + l.levelsModSpec() + .toLogctlModSpec() + .equals("fatal=on,error=on,warning=on,info=off,event=on,config=on,debug=off,spam=off"))) + .count()); + + String localhostConfigId = "hosts/myhost0"; + SentinelConfig sentinelConfig = vespaModel.getConfig(SentinelConfig.class, localhostConfigId); + System.out.println(sentinelConfig); + assertEquals(4, getConfigForService("container", sentinelConfig).logctl().size()); + assertEquals(4, getConfigForService("metricsproxy-container", sentinelConfig).logctl().size()); + } + + private SentinelConfig.Service getConfigForService(String serviceName, SentinelConfig config) { + return config.service().stream().filter(service -> service.name().equals(serviceName)).findFirst().get(); + } + } diff --git a/configd/src/apps/sentinel/logctl.cpp b/configd/src/apps/sentinel/logctl.cpp index 057178cec2e..59c02b57344 100644 --- a/configd/src/apps/sentinel/logctl.cpp +++ b/configd/src/apps/sentinel/logctl.cpp @@ -5,6 +5,7 @@ #include <sys/types.h> #include <sys/wait.h> #include <unistd.h> +#include <fcntl.h> #include <cstring> #include <vespa/log/log.h> @@ -18,28 +19,40 @@ void justRunLogctl(const char *cspec, const char *lspec) pid_t pid = fork(); if (pid == 0) { LOG(debug, "running '%s' '%s' '%s'", progName, cspec, lspec); + int devnull = open("/dev/null", O_WRONLY); + if (devnull > 1) { + dup2(devnull, 1); + close(devnull); + } int rv = execlp(progName, progName, "-c", cspec, lspec, nullptr); if (rv != 0) { LOG(warning, "execlp of '%s' failed: %s", progName, strerror(errno)); } } else if (pid > 0) { - int wstatus = 0; - pid_t got = waitpid(pid, &wstatus, 0); - if (got == pid) { - if (WIFEXITED(wstatus)) { - int exitCode = WEXITSTATUS(wstatus); - if (exitCode != 0) { - LOG(warning, "running '%s' failed (exit code %d)", progName, exitCode); + bool again; + do { + again = false; + int wstatus = 0; + pid_t got = waitpid(pid, &wstatus, 0); + if (got == pid) { + again = false; + if (WIFEXITED(wstatus)) { + int exitCode = WEXITSTATUS(wstatus); + if (exitCode != 0) { + LOG(warning, "running '%s' failed (exit code %d)", progName, exitCode); + } + } else if (WIFSIGNALED(wstatus)) { + int termSig = WTERMSIG(wstatus); + LOG(warning, "running '%s' failed (got signal %d)", progName, termSig); + } else { + LOG(warning, "'%s' failure (wait status was %d)", progName, wstatus); } - } else if (WIFSIGNALED(wstatus)) { - int termSig = WTERMSIG(wstatus); - LOG(warning, "running '%s' failed (got signal %d)", progName, termSig); + } else if (errno == EINTR) { + again = true; } else { - LOG(warning, "'%s' failure (wait status was %d)", progName, wstatus); + LOG(error, "waitpid() failed: %s", strerror(errno)); } - } else { - LOG(error, "waitpid() failed: %s", strerror(errno)); - } + } while (again); } else { LOG(error, "fork() failed: %s", strerror(errno)); } diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index 31f2bfc281d..2810284b3a3 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -2,7 +2,7 @@ namespace=vespa.config.search attribute[].name string -attribute[].datatype enum { STRING, BOOL, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE +attribute[].datatype enum { STRING, BOOL, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, RAW, NONE } default=NONE attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE attribute[].dictionary.type enum { BTREE, HASH, BTREE_AND_HASH } default = BTREE attribute[].dictionary.match enum { CASE_SENSITIVE, CASE_INSENSITIVE, CASED, UNCASED } default=UNCASED diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java index 61c0c17264c..7920bbed763 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.metrics; import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.metrics.ClusterControllerMetrics; import com.yahoo.metrics.ContainerMetrics; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -137,11 +138,11 @@ public class ClusterDeploymentMetricsRetriever { case VESPA_DISTRIBUTOR -> optionalDouble(values.field("vds.distributor.docsstored.average")) .ifPresent(docCount -> aggregator.get().addDocumentCount(docCount)); case VESPA_CONTAINER_CLUSTERCONTROLLER -> - optionalDouble(values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_MEMORY_UTILIZATION.max())).ifPresent(memoryUtil -> + optionalDouble(values.field(ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION.max())).ifPresent(memoryUtil -> aggregator.get() - .addMemoryUsage(memoryUtil, values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MEMORY_LIMIT.last()).asDouble()) - .addDiskUsage(values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_DISK_UTILIZATION.max()).asDouble(), - values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_DISK_LIMIT.last()).asDouble())); + .addMemoryUsage(memoryUtil, values.field(ClusterControllerMetrics.RESOURCE_USAGE_MEMORY_LIMIT.last()).asDouble()) + .addDiskUsage(values.field(ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION.max()).asDouble(), + values.field(ClusterControllerMetrics.RESOURCE_USAGE_DISK_LIMIT.last()).asDouble())); } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index 6753fed2c5a..a4eaab2e8f8 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -22,10 +22,10 @@ class MetricDefinitions { static final String STATUS_CODE_DIMENSION = "statusCode"; static final String NUM_OPEN_CONNECTIONS = ContainerMetrics.SERVER_NUM_OPEN_CONNECTIONS.baseName(); - static final String NUM_CONNECTIONS_OPEN_MAX = "serverConnectionsOpenMax"; - static final String CONNECTION_DURATION_MAX = "serverConnectionDurationMax"; - static final String CONNECTION_DURATION_MEAN = "serverConnectionDurationMean"; - static final String CONNECTION_DURATION_STD_DEV = "serverConnectionDurationStdDev"; + static final String NUM_CONNECTIONS_OPEN_MAX = ContainerMetrics.SERVER_CONNECTIONS_OPEN_MAX.baseName(); + static final String CONNECTION_DURATION_MAX = ContainerMetrics.SERVER_CONNECTION_DURATION_MAX.baseName(); + static final String CONNECTION_DURATION_MEAN = ContainerMetrics.SERVER_CONNECTION_DURATION_MEAN.baseName(); + static final String CONNECTION_DURATION_STD_DEV = ContainerMetrics.SERVER_CONNECTION_DURATION_STD_DEV.baseName(); static final String NUM_PREMATURELY_CLOSED_CONNECTIONS = ContainerMetrics.JDISC_HTTP_REQUEST_PREMATURELY_CLOSED.baseName(); static final String REQUESTS_PER_CONNECTION = ContainerMetrics.JDISC_HTTP_REQUEST_REQUESTS_PER_CONNECTION.baseName(); @@ -36,16 +36,16 @@ class MetricDefinitions { /* For historical reasons, these are all aliases for the same metric. 'jdisc.http' should ideally be the only one. */ static final String JDISC_HTTP_REQUESTS = ContainerMetrics.JDISC_HTTP_REQUESTS.baseName(); - static final String NUM_REQUESTS = "serverNumRequests"; + static final String NUM_REQUESTS = ContainerMetrics.SERVER_NUM_REQUESTS.baseName(); - static final String NUM_SUCCESSFUL_RESPONSES = "serverNumSuccessfulResponses"; - static final String NUM_FAILED_RESPONSES = "serverNumFailedResponses"; - static final String NUM_SUCCESSFUL_WRITES = "serverNumSuccessfulResponseWrites"; - static final String NUM_FAILED_WRITES = "serverNumFailedResponseWrites"; + static final String NUM_SUCCESSFUL_RESPONSES = ContainerMetrics.SERVER_NUM_SUCCESSFUL_RESPONSES.baseName(); + static final String NUM_FAILED_RESPONSES = ContainerMetrics.SERVER_NUM_FAILED_RESPONSES.baseName(); + static final String NUM_SUCCESSFUL_WRITES = ContainerMetrics.SERVER_NUM_SUCCESSFUL_RESPONSE_WRITES.baseName(); + static final String NUM_FAILED_WRITES = ContainerMetrics.SERVER_NUM_FAILED_RESPONSE_WRITES.baseName(); - static final String TOTAL_SUCCESSFUL_LATENCY = "serverTotalSuccessfulResponseLatency"; - static final String TOTAL_FAILED_LATENCY = "serverTotalFailedResponseLatency"; - static final String TIME_TO_FIRST_BYTE = "serverTimeToFirstByte"; + static final String TOTAL_SUCCESSFUL_LATENCY = ContainerMetrics.SERVER_TOTAL_SUCCESFUL_RESPONSE_LATENCY.baseName(); + static final String TOTAL_FAILED_LATENCY = ContainerMetrics.SERVER_TOTAL_FAILED_RESPONSE_LATENCY.baseName(); + static final String TIME_TO_FIRST_BYTE = ContainerMetrics.SERVER_TIME_TO_FIRST_BYTE.baseName(); static final String RESPONSES_1XX = ContainerMetrics.HTTP_STATUS_1XX.baseName(); static final String RESPONSES_2XX = ContainerMetrics.HTTP_STATUS_2XX.baseName(); @@ -53,7 +53,7 @@ class MetricDefinitions { static final String RESPONSES_4XX = ContainerMetrics.HTTP_STATUS_4XX.baseName(); static final String RESPONSES_5XX = ContainerMetrics.HTTP_STATUS_5XX.baseName(); - static final String STARTED_MILLIS = "serverStartedMillis"; + static final String STARTED_MILLIS = ContainerMetrics.SERVER_STARTED_MILLIS.baseName(); static final String URI_LENGTH = ContainerMetrics.JDISC_HTTP_REQUEST_URI_LENGTH.baseName(); static final String CONTENT_SIZE = ContainerMetrics.JDISC_HTTP_REQUEST_CONTENT_SIZE.baseName(); diff --git a/container-core/src/main/java/com/yahoo/metrics/ClusterControllerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/ClusterControllerMetrics.java new file mode 100644 index 00000000000..fabfd5504f7 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/ClusterControllerMetrics.java @@ -0,0 +1,53 @@ +package com.yahoo.metrics; + +/** + * @author yngve + */ +public enum ClusterControllerMetrics implements VespaMetrics { + + DOWN_COUNT("cluster-controller.down.count", Unit.NODE, "Number of content nodes down"), + INITIALIZING_COUNT("cluster-controller.initializing.count", Unit.NODE, "Number of content nodes initializing"), + MAINTENANCE_COUNT("cluster-controller.maintenance.count", Unit.NODE, "Number of content nodes in maintenance"), + RETIRED_COUNT("cluster-controller.retired.count", Unit.NODE, "Number of content nodes that are retired"), + STOPPING_COUNT("cluster-controller.stopping.count", Unit.NODE, "Number of content nodes currently stopping"), + UP_COUNT("cluster-controller.up.count", Unit.NODE, "Number of content nodes up"), + CLUSTER_STATE_CHANGE_COUNT("cluster-controller.cluster-state-change.count", Unit.NODE, "Number of nodes changing state"), + BUSY_TICK_TIME_MS("cluster-controller.busy-tick-time-ms", Unit.MILLISECOND, "Time busy"), + IDLE_TICK_TIME_MS("cluster-controller.idle-tick-time-ms", Unit.MILLISECOND, "Time idle"), + WORK_MS("cluster-controller.work-ms", Unit.MILLISECOND, "Time used for actual work"), + IS_MASTER("cluster-controller.is-master", Unit.BINARY, "1 if this cluster controller is currently the master, or 0 if not"), + REMOTE_TASK_QUEUE_SIZE("cluster-controller.remote-task-queue.size", Unit.OPERATION, "Number of remote tasks queued"), + // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. + // DO NOT RELY ON THIS METRIC YET. + NODE_EVENT_COUNT("cluster-controller.node-event.count", Unit.OPERATION, "Number of node events"), + RESOURCE_USAGE_NODES_ABOVE_LIMIT("cluster-controller.resource_usage.nodes_above_limit", Unit.NODE, "The number of content nodes above resource limit, blocking feed"), + RESOURCE_USAGE_MAX_MEMORY_UTILIZATION("cluster-controller.resource_usage.max_memory_utilization", Unit.FRACTION, "Current memory utilisation, per content node"), + RESOURCE_USAGE_MAX_DISK_UTILIZATION("cluster-controller.resource_usage.max_disk_utilization", Unit.FRACTION, "Current disk space utilisation, per content node"), + RESOURCE_USAGE_MEMORY_LIMIT("cluster-controller.resource_usage.memory_limit", Unit.FRACTION, "Disk space limit as a fraction of available disk space"), + RESOURCE_USAGE_DISK_LIMIT("cluster-controller.resource_usage.disk_limit", Unit.FRACTION, "Memory space limit as a fraction of available memory"), + REINDEXING_PROGRESS("reindexing.progress", Unit.FRACTION, "Re-indexing progress"); + + + private final String name; + private final Unit unit; + private final String description; + + ClusterControllerMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java index ed1d6f7a001..c443c387381 100644 --- a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java @@ -158,42 +158,47 @@ public enum ContainerMetrics implements VespaMetrics { ERROR_UNSPECIFIED("error.unspecified", Unit.OPERATION, "Requests that failed for an unspecified reason"), ERROR_UNHANDLED_EXCEPTION("error.unhandled_exception", Unit.OPERATION, "Requests that failed due to an unhandled exception"), - // Deprecated metrics. TODO: Remove on Vespa 9. + // Deprecated metrics. TODO: Remove in Vespa 9. SERVER_REJECTED_REQUESTS("serverRejectedRequests", Unit.OPERATION, "Deprecated. Use jdisc.thread_pool.rejected_tasks instead."), SERVER_THREAD_POOL_SIZE("serverThreadPoolSize", Unit.THREAD, "Deprecated. Use jdisc.thread_pool.size instead."), SERVER_ACTIVE_THREADS("serverActiveThreads", Unit.THREAD, "Deprecated. Use jdisc.thread_pool.active_threads instead."), - - // Metrics from the cluster controller - CLUSTER_CONTROLLER_DOWN_COUNT("cluster-controller.down.count", Unit.NODE, "Number of content nodes down"), - CLUSTER_CONTROLLER_INITIALIZING_COUNT("cluster-controller.initializing.count", Unit.NODE, "Number of content nodes initializing"), - CLUSTER_CONTROLLER_MAINTENANCE_COUNT("cluster-controller.maintenance.count", Unit.NODE, "Number of content nodes in maintenance"), - CLUSTER_CONTROLLER_RETIRED_COUNT("cluster-controller.retired.count", Unit.NODE, "Number of content nodes that are retired"), - CLUSTER_CONTROLLER_STOPPING_COUNT("cluster-controller.stopping.count", Unit.NODE, "Number of content nodes currently stopping"), - CLUSTER_CONTROLLER_UP_COUNT("cluster-controller.up.count", Unit.NODE, "Number of content nodes up"), - CLUSTER_CONTROLLER_CLUSTER_STATE_CHANGE_COUNT("cluster-controller.cluster-state-change.count", Unit.NODE, "Number of nodes changing state"), - CLUSTER_CONTROLLER_BUSY_TICK_TIME_MS("cluster-controller.busy-tick-time-ms", Unit.MILLISECOND, "Time busy"), - CLUSTER_CONTROLLER_IDLE_TICK_TIME_MS("cluster-controller.idle-tick-time-ms", Unit.MILLISECOND, "Time idle"), - CLUSTER_CONTROLLER_WORK_MS("cluster-controller.work-ms", Unit.MILLISECOND, "Time used for actual work"), - CLUSTER_CONTROLLER_IS_MASTER("cluster-controller.is-master", Unit.BINARY, "1 if this cluster controller is currently the master, or 0 if not"), - CLUSTER_CONTROLLER_REMOTE_TASK_QUEUE_SIZE("cluster-controller.remote-task-queue.size", Unit.OPERATION, "Number of remote tasks queued"), - // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. - // DO NOT RELY ON THIS METRIC YET. - CLUSTER_CONTROLLER_NODE_EVENT_COUNT("cluster-controller.node-event.count", Unit.OPERATION, "Number of node events"), - CLUSTER_CONTROLLER_RESOURCE_USAGE_NODES_ABOVE_LIMIT("cluster-controller.resource_usage.nodes_above_limit", Unit.NODE, "The number of content nodes above resource limit, blocking feed"), - CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_MEMORY_UTILIZATION("cluster-controller.resource_usage.max_memory_utilization", Unit.FRACTION, "Current memory utilisation, per content node"), - CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_DISK_UTILIZATION("cluster-controller.resource_usage.max_disk_utilization", Unit.FRACTION, "Current disk space utilisation, per content node"), - CLUSTER_CONTROLLER_RESOURCE_USAGE_MEMORY_LIMIT("cluster-controller.resource_usage.memory_limit", Unit.FRACTION, "Disk space limit as a fraction of available disk space"), - CLUSTER_CONTROLLER_RESOURCE_USAGE_DISK_LIMIT("cluster-controller.resource_usage.disk_limit", Unit.FRACTION, "Memory space limit as a fraction of available memory"), - CLUSTER_CONTROLLER_REINDEXING_PROGRESS("reindexing.progress", Unit.FRACTION, "Re-indexing progress"), - // Java (JRT) TLS metrics JRT_TRANSPORT_TLS_CERTIFICATE_VERIFICATION_FAILURES("jrt.transport.tls-certificate-verification-failures", Unit.FAILURE, "TLS certificate verification failures"), JRT_TRANSPORT_PEER_AUTHORIZATION_FAILURES("jrt.transport.peer-authorization-failures", Unit.FAILURE, "TLS peer authorization failures"), JRT_TRANSPORT_SERVER_TLS_CONNECIONTS_ESTABLISHED("jrt.transport.server.tls-connections-established", Unit.CONNECTION, "TLS server connections established"), JRT_TRANSPORT_CLIENT_TLS_CONNECTIONS_ESTABLISHED("jrt.transport.client.tls-connections-established", Unit.CONNECTION, "TLS client connections established"), JRT_TRANSPORT_SERVER_UNENCRYPTED_CONNECTIONS_ESTABLISHED("jrt.transport.server.unencrypted-connections-established", Unit.CONNECTION, "Unencrypted server connections established"), - JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED("jrt.transport.client.unencrypted-connections-established", Unit.CONNECTION, "Unencrypted client connections established"); + JRT_TRANSPORT_CLIENT_UNENCRYPTED_CONNECTIONS_ESTABLISHED("jrt.transport.client.unencrypted-connections-established", Unit.CONNECTION, "Unencrypted client connections established"), + + MAX_QUERY_LATENCY("max_query_latency", Unit.MILLISECOND, "Deprecated. Use query_latency.max instead"), // TODO: Remove in Vespa 9 + MEAN_QUERY_LATENCY("mean_query_latency", Unit.MILLISECOND, "Deprecated. Use the expression (query_latency.sum / query_latency.count) instead"),// TODO: Remove in Vespa 9 + + + // Metrics defined in com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java + JDISC_HTTP_FILTER_ATHENZ_ACCEPTED_REQUESTS("jdisc.http.filter.athenz.accepted_requests", Unit.REQUEST, "Number of requests accepted by the AthenzAuthorization filter"), + JDISC_HTTP_FILTER_ATHENZ_REJECTED_REQUESTS("jdisc.http.filter.athenz.rejected_requests", Unit.REQUEST, "Number of requests rejected by the AthenzAuthorization filter"), + + + // Metrics defined in com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java + SERVER_CONNECTIONS_OPEN_MAX("serverConnectionsOpenMax", Unit.CONNECTION, "Maximum number of open connections"), + SERVER_CONNECTION_DURATION_MAX("serverConnectionDurationMax", Unit.MILLISECOND, "Longest duration a connection is kept open"), + + SERVER_CONNECTION_DURATION_MEAN("serverConnectionDurationMean", Unit.MILLISECOND, "Average duration a connection is kept open"), + SERVER_CONNECTION_DURATION_STD_DEV("serverConnectionDurationStdDev", Unit.MILLISECOND, "Standard deviation of open connection duration"), + SERVER_NUM_REQUESTS("serverNumRequests", Unit.REQUEST, "Number of requests"), + SERVER_NUM_SUCCESSFUL_RESPONSES("serverNumSuccessfulResponses", Unit.REQUEST, "Number of successful responses"), + SERVER_NUM_FAILED_RESPONSES("serverNumFailedResponses", Unit.REQUEST, "Number of failed responses"), + + SERVER_NUM_SUCCESSFUL_RESPONSE_WRITES("serverNumSuccessfulResponseWrites", Unit.REQUEST, "Number of successful response writes"), + SERVER_NUM_FAILED_RESPONSE_WRITES("serverNumFailedResponseWrites", Unit.REQUEST, "Number of failed response writes"), + + SERVER_TOTAL_SUCCESFUL_RESPONSE_LATENCY("serverTotalSuccessfulResponseLatency", Unit.MILLISECOND, "Total duration for execution of successful responses"), + SERVER_TOTAL_FAILED_RESPONSE_LATENCY("serverTotalFailedResponseLatency", Unit.MILLISECOND, "Total duration for execution of failed responses"), + SERVER_TIME_TO_FIRST_BYTE("serverTimeToFirstByte", Unit.MILLISECOND, "Time from request has been received by the server until the first byte is returned to the client"), + + SERVER_STARTED_MILLIS("serverStartedMillis", Unit.MILLISECOND, "Time since the service was started"); + private final String name; diff --git a/container-core/src/main/java/com/yahoo/metrics/VespaMetrics.java b/container-core/src/main/java/com/yahoo/metrics/VespaMetrics.java index 5b23dfa0772..c9d1f058619 100644 --- a/container-core/src/main/java/com/yahoo/metrics/VespaMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/VespaMetrics.java @@ -3,7 +3,7 @@ package com.yahoo.metrics; /** * @author gjoranv */ -interface VespaMetrics { +public interface VespaMetrics { String baseName(); Unit unit(); diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 552c6e56144..c13bdff1f50 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -57,12 +57,12 @@ import static com.yahoo.container.protect.Error.UNSPECIFIED; public class StatisticsSearcher extends Searcher { private static final CompoundName IGNORE_QUERY = new CompoundName("metrics.ignore"); - private static final String MAX_QUERY_LATENCY_METRIC = "max_query_latency"; + private static final String MAX_QUERY_LATENCY_METRIC = ContainerMetrics.MAX_QUERY_LATENCY.baseName(); private static final String EMPTY_RESULTS_METRIC = ContainerMetrics.EMPTY_RESULTS.baseName(); private static final String HITS_PER_QUERY_METRIC = ContainerMetrics.HITS_PER_QUERY.baseName(); private static final String TOTALHITS_PER_QUERY_METRIC = ContainerMetrics.TOTAL_HITS_PER_QUERY.baseName(); private static final String FAILED_QUERIES_METRIC = ContainerMetrics.FAILED_QUERIES.baseName(); - private static final String MEAN_QUERY_LATENCY_METRIC = "mean_query_latency"; + private static final String MEAN_QUERY_LATENCY_METRIC = ContainerMetrics.MEAN_QUERY_LATENCY.baseName(); private static final String QUERY_LATENCY_METRIC = ContainerMetrics.QUERY_LATENCY.baseName(); private static final String QUERY_TIMEOUT_METRIC = ContainerMetrics.QUERY_TIMEOUT.baseName(); private static final String QUERY_HIT_OFFSET_METRIC = ContainerMetrics.QUERY_HIT_OFFSET.baseName(); diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java index 2dc1f549e07..2eee55e4102 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java @@ -7,6 +7,7 @@ import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.athenz.RequestResourceMapper.ResourceNameAndAction; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; +import com.yahoo.metrics.ContainerMetrics; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -48,8 +49,8 @@ public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase { public static final String MATCHED_ROLE_ATTRIBUTE = ATTRIBUTE_PREFIX + ".matched-role"; public static final String IDENTITY_NAME_ATTRIBUTE = ATTRIBUTE_PREFIX + ".identity-name"; public static final String MATCHED_CREDENTIAL_TYPE_ATTRIBUTE = ATTRIBUTE_PREFIX + ".credentials-type"; - private static final String ACCEPTED_METRIC_NAME = "jdisc.http.filter.athenz.accepted_requests"; - private static final String REJECTED_METRIC_NAME = "jdisc.http.filter.athenz.rejected_requests"; + private static final String ACCEPTED_METRIC_NAME = ContainerMetrics.JDISC_HTTP_FILTER_ATHENZ_ACCEPTED_REQUESTS.baseName(); + private static final String REJECTED_METRIC_NAME = ContainerMetrics.JDISC_HTTP_FILTER_ATHENZ_REJECTED_REQUESTS.baseName(); private static final Logger log = Logger.getLogger(AthenzAuthorizationFilter.class.getName()); diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/BindingExtractor.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/BindingExtractor.java index 6b1f60df6f4..126e9f9f4e6 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/BindingExtractor.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/BindingExtractor.java @@ -7,6 +7,9 @@ import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.tensor.functions.Generate; +import com.yahoo.tensor.functions.Slice; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -45,6 +48,11 @@ class BindingExtractor { arguments.addAll(other.arguments); onnxModelsInUse.putAll(other.onnxModelsInUse); } + + void removeTarget(String name) { + bindTargets.remove(name); + arguments.remove(name); + } } private final Map<FunctionReference, FunctionInfo> functionsInfo = new LinkedHashMap<>(); @@ -83,6 +91,26 @@ class BindingExtractor { } return result; } + else if (node instanceof TensorFunctionNode tfn) { + for (ExpressionNode child : tfn.children()) { + result.merge(extractBindTargets(child)); + } + var f = tfn.function(); + if (f instanceof Generate) { + var tt = f.type(null); + for (var dim : tt.dimensions()) { + result.removeTarget(dim.name()); + } + } + else if (f instanceof Slice<?> s) { + for (var selectorFunc : s.selectorFunctions()) { + if (selectorFunc instanceof TensorFunctionNode.ExpressionTensorFunction expr) { + result.merge(extractBindTargets(expr.wrappedExpression())); + } + } + } + return result; + } else if (isOnnx(node)) { return extractOnnxTargets(node); } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java index 666c3a103b5..a34898fe4e9 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java @@ -109,7 +109,7 @@ class FunctionReference { /** * Returns a function reference from the given return type serial form, - * or empty if the string is not a valid function return typoe serial form + * or empty if the string is not a valid function return type serial form */ static Optional<FunctionReference> fromReturnTypeSerial(String serialForm) { Matcher expressionMatcher = returnTypePattern.matcher(serialForm); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index ceedbcf89c2..313cf45e1ee 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -34,7 +34,7 @@ public interface LoadBalancerService { void remove(LoadBalancer loadBalancer); /** Returns the protocol supported by this load balancer service */ - Protocol protocol(); + Protocol protocol(boolean enclave); /** Returns whether load balancers created by this service can forward traffic to given node and cluster type */ boolean supports(NodeType nodeType, ClusterSpec.Type clusterType); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 1f354dc7081..751f3d46059 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -5,7 +5,6 @@ import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ZoneEndpoint; import java.util.Collections; import java.util.HashMap; @@ -45,7 +44,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public Protocol protocol() { + public Protocol protocol(boolean enclave) { return Protocol.ipv4; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 22367f72666..f9f26852b0d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -57,7 +57,8 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public Protocol protocol() { + public Protocol protocol(boolean enclave) { + if (enclave) throw new IllegalArgumentException("enclave is not supported with " + getClass()); return Protocol.dualstack; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index cc63ab28a70..04f64b070b3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -332,7 +332,7 @@ public class LoadBalancerProvisioner { private Set<String> reachableIpAddresses(Node node) { Set<String> reachable = new LinkedHashSet<>(node.ipConfig().primary()); // Remove addresses unreachable by the load balancer service - switch (service.protocol()) { + switch (service.protocol(node.cloudAccount().isEnclave(nodeRepository.zone()))) { case ipv4 -> reachable.removeIf(IP::isV6); case ipv6 -> reachable.removeIf(IP::isV4); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index 1a6058bed39..98effdca182 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -41,7 +41,7 @@ public class SharedLoadBalancerServiceTest { @Test public void test_protocol() { - assertEquals(LoadBalancerService.Protocol.dualstack, loadBalancerService.protocol()); + assertEquals(LoadBalancerService.Protocol.dualstack, loadBalancerService.protocol(false)); } } diff --git a/screwdriver.yaml b/screwdriver.yaml index c9de232d893..99d5b6e14c5 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -371,6 +371,7 @@ jobs: screwdriver.cd/buildPeriodically: H 0 * * * steps: - install: | + dnf install -y dnf-plugins-core dnf config-manager --add-repo https://raw.githubusercontent.com/vespa-engine/vespa/master/dist/vespa-engine.repo dnf config-manager --enable powertools dnf install -y epel-release diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 8dc2551d96b..c9f3a1c1de8 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -11,6 +11,7 @@ #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> +#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> #include <vespa/searchcore/proton/initializer/task_runner.h> @@ -240,7 +241,7 @@ struct ParallelAttributeManager { InitializerTask::SP documentMetaStoreInitTask; std::shared_ptr<bucketdb::BucketDBOwner> bucketDbOwner; - DocumentMetaStore::SP documentMetaStore; + std::shared_ptr<DocumentMetaStore> documentMetaStore; AllocStrategy alloc_strategy; bool fastAccessAttributesOnly; std::shared_ptr<AttributeManager::SP> mgr; diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp index 8c90b189b69..037bf884e6b 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchcore/proton/common/attribute_updater.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/reference_attribute.h> +#include <vespa/searchlib/attribute/single_raw_attribute.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> @@ -13,6 +14,7 @@ #include <vespa/document/fieldvalue/bytefieldvalue.h> #include <vespa/document/fieldvalue/floatfieldvalue.h> #include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/fieldvalue/rawfieldvalue.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> @@ -34,6 +36,7 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> @@ -49,6 +52,7 @@ using search::attribute::CollectionType; using search::attribute::Config; using search::attribute::Reference; using search::attribute::ReferenceAttribute; +using search::attribute::SingleRawAttribute; using search::tensor::ITensorAttribute; using search::tensor::DenseTensorAttribute; using search::tensor::SerializedFastValueAttribute; @@ -58,6 +62,18 @@ using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; +namespace { + +std::vector<char> as_vector(vespalib::stringref value) { + return {value.data(), value.data() + value.size()}; +} + +std::vector<char> as_vector(vespalib::ConstArrayRef<char> value) { + return {value.data(), value.data() + value.size()}; +} + +} + namespace search { using AttributePtr = AttributeVector::SP; @@ -74,6 +90,7 @@ makeDocumentTypeRepo() .addField("int", DataType::T_INT) .addField("float", DataType::T_FLOAT) .addField("string", DataType::T_STRING) + .addField("raw", DataType::T_RAW) .addField("aint", Array(DataType::T_INT)) .addField("afloat", Array(DataType::T_FLOAT)) .addField("astring", Array(DataType::T_STRING)) @@ -121,6 +138,10 @@ struct Fixture { applyValueUpdate(vec, 3, std::make_unique<ClearValueUpdate>()); applyValueUpdate(vec, 4, std::make_unique<MapValueUpdate>(std::move(copyOfFirst), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10))); } + + void applyValue(AttributeVector& vec, uint32_t docid, std::unique_ptr<FieldValue> value) { + search::AttributeUpdater::handleValue(vec, docid, *value); + } }; template <typename T, typename VectorType> @@ -173,7 +194,6 @@ check(const AttributePtr &vec, uint32_t docId, const std::vector<T> &values) return true; } - GlobalId toGid(vespalib::stringref docId) { return DocumentId(docId).getGlobalId(); } @@ -254,6 +274,20 @@ TEST_F("require that single attributes are updated", Fixture) TEST_DO(assertRef(*vec, doc1, 1)); TEST_DO(assertNoRef(*vec, 2)); } + { + BasicType bt(BasicType::RAW); + vespalib::string first_backing("first"); + vespalib::ConstArrayRef<char> first(first_backing.data(), first_backing.size()); + AttributePtr vec = create<vespalib::ConstArrayRef<char>, SingleRawAttribute>(4, first, 0, "in1/raw", Config(bt, ct)); + f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<RawFieldValue>("second"))); + f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); + f.applyValue(*vec, 3, std::make_unique<RawFieldValue>("third")); + EXPECT_EQUAL(4u, vec->getNumDocs()); + EXPECT_EQUAL(as_vector("second"), as_vector(vec->get_raw(0))); + EXPECT_EQUAL(as_vector("first"), as_vector(vec->get_raw(1))); + EXPECT_EQUAL(as_vector(""), as_vector(vec->get_raw(2))); + EXPECT_EQUAL(as_vector("third"), as_vector(vec->get_raw(3))); + } } TEST_F("require that array attributes are updated", Fixture) diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index 2e217a3f6a8..8185f8a30be 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -101,6 +101,7 @@ using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; using vespalib::eval::Value; +using proton::documentmetastore::IStore; using namespace document::config_builder; using namespace search::index; @@ -296,7 +297,7 @@ struct Fixture { const GlobalId &gid; BucketId bucket_id; Timestamp timestamp; - DocumentMetaStore::DocId lid; + IStore::DocId lid; MyDocumentStore doc_store; search::AttributeManager attr_manager; Schema schema; @@ -365,11 +366,10 @@ struct Fixture { _dtName(doc_type_name), _retriever() { - using Result = DocumentMetaStore::Result; meta_store.constructFreeList(); - Result inspect = meta_store.get().inspect(gid, 0u); + IStore::Result inspect = meta_store.get().inspect(gid, 0u); uint32_t docSize = 1; - Result putRes(meta_store.get().put(gid, bucket_id, timestamp, docSize, inspect.getLid(), 0u)); + IStore::Result putRes(meta_store.get().put(gid, bucket_id, timestamp, docSize, inspect.getLid(), 0u)); meta_store.get().commit(search::CommitParam(0)); lid = putRes.getLid(); ASSERT_TRUE(putRes.ok()); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 375ff0f2012..e85e6c58e11 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -129,10 +129,6 @@ indexing.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart ## Control io options during read both under dump and fusion. indexing.read.io enum {NORMAL, DIRECTIO} default=DIRECTIO restart -## Overrides the number of threads used for writing fields across all document dbs. -## See feeding.concurrency for details. -indexing.threads int default=1 restart - ## Option to specify what is most important during indexing. ## This is experimental and will most likely be temporary. indexing.optimize enum {LATENCY, THROUGHPUT, ADAPTIVE} default=THROUGHPUT restart @@ -141,9 +137,6 @@ indexing.optimize enum {LATENCY, THROUGHPUT, ADAPTIVE} default=THROUGHPUT restar ## indexing threads. Only used when visibility delay is zero. indexing.tasklimit int default=-1000 -## Deprecated and ignored, will soon go away -indexing.semiunboundtasklimit int default = 1000 - ## Kind of watermark for when to activate extra manpower ## Utilized if optimize is set to either THROUGHPUT or ADAPTIVE indexing.kind_of_watermark int default = 0 restart @@ -151,7 +144,6 @@ indexing.kind_of_watermark int default = 0 restart ## Controls minimum reaction time in seconds if using THROUGHPUT indexing.reactiontime double default = 0.001 restart - ## How long a freshly loaded index shall be warmed up ## before being used for serving index.warmup.time double default=0.0 restart @@ -169,10 +161,6 @@ index.maxflushed int default=2 ## Setting to 1 will force an immediate fusion. index.maxflushedretired int default=20 -## How much memory is set aside for caching. -## Now only used for caching of dictionary lookups. -index.cache.size long default=0 restart - ## Control io options during flushing of attributes. attribute.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart @@ -180,10 +168,12 @@ attribute.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart search.mmap.options[] enum {POPULATE, HUGETLB} restart ## Advise to give to os when mapping memory. +## TODO Check if default should be random search.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart ## Max number of threads allowed to handle large queries concurrently ## Positive number means there is a limit, 0 or negative means no limit. +## TODO Check if ever used in config. search.memory.limiter.maxthreads int default=0 ## Minimum coverage of corpus to postprocess before applying above concurrency limit. @@ -210,17 +200,6 @@ distribution.searchablecopies long default=1 ## Negative numbers are a percentage of memory. summary.cache.maxbytes long default=-4 -## Include visits in the cache, if the visitoperation allows it. -## This will enable another separate cache of summary.cache.maxbytes size. -## IGNORED and DEPRECATED Will go away soon -summary.cache.allowvisitcaching bool default=true - -## Control number of cache entries preallocated. -## Default is no preallocation. -## Can be set to a higher number to avoid resizing. -## IGNORED and DEPRECATED Will go away soon -summary.cache.initialentries long default=0 restart - ## Control compression type of the summary while in the cache. summary.cache.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 @@ -234,6 +213,7 @@ summary.cache.update_strategy enum {INVALIDATE, UPDATE} default=INVALIDATE ## Control compression type of the summary while in memory during compaction ## NB So far only stragey=LOG honours it. +## TODO Use same as for store (chunk.compression). summary.log.compact.compression.type enum {NONE, LZ4, ZSTD} default=ZSTD ## Control compression level of the summary while in memory during compaction @@ -253,14 +233,11 @@ summary.log.chunk.compression.level int default=9 ## Max size in bytes per chunk. summary.log.chunk.maxbytes int default=65536 -## Skip crc32 check on read. -## IGNORED and DEPRECATED Will go away soon -summary.log.chunk.skipcrconread bool default=false - ## Max size per summary file. summary.log.maxfilesize long default=1000000000 ## Max number of lid entries per file +## TODO Decide based on memory on node. summary.log.maxnumlids int default=40000000 ## Max disk bloat factor. This will trigger compacting. @@ -279,6 +256,7 @@ summary.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO ## Control io options during read of stored documents. ## All summary.read options will take effect immediately on new files written. ## On old files it will take effect either upon compact or on restart. +## TODO Default is probably DIRECTIO summary.read.io enum {NORMAL, DIRECTIO, MMAP } default=MMAP restart ## Multiple optional options for use with mmap @@ -331,6 +309,7 @@ documentdb[].allocation.active_buffers_ratio double default=0.1 periodic.interval double default=3600.0 ## Connect spec for transactionlog server. +## TODO Consider not using RPC at all tlsspec string default="tcp/localhost:13700" restart ## ConfigId for transactionlogserver @@ -424,6 +403,7 @@ visit.ignoremaxbytes bool default=true ## Number of initializer threads used for loading structures from disk at proton startup. ## The threads are shared between document databases when value is larger than 0. ## When set to 0 (default) we use 1 separate thread per document database. +## TODO Consider if really necessary, could be automatic. initialize.threads int default = 0 ## Portion of max address space used in components in attribute vectors @@ -457,10 +437,12 @@ hwinfo.disk.writespeed double default = 200.0 restart ## Amount of data to write to temporary file when sampling disk write speed. ## Default is 1 GiB. +## TODO Check if still in use hwinfo.disk.samplewritesize long default = 1073741824 restart ## Minimum write speed needed to avoid disk being considered slow. ## Unit is MiB/s, default is 100.0 MiB/s. +## TODO Check if still in use hwinfo.disk.slowwritespeedlimit double default = 100.0 restart ## The size of physical memory (in bytes) available to proton. @@ -496,6 +478,7 @@ feeding.niceness double default = 0.0 restart ## This limit is only considered when executing tasks for handling external feed operations. ## In that case the calling thread (persistence thread) is blocked until the master thread has capacity to handle more tasks. ## When this limit is set to 0 it is ignored. +## TODO Check if still in use feeding.master_task_limit int default = 0 ## Adjustment to resource limit when determining if maintenance jobs can run. @@ -513,6 +496,45 @@ maintenancejobs.maxoutstandingmoveops int default=100 ## in depth understanding is present. bucketdb.checksumtype enum {LEGACY, XXHASH64} default = LEGACY restart +## Chooses the throttling policy used to control the window size +## of the SharedOperationThrottler component used by the transaction log replay feed state. +replay_throttling_policy.type enum { UNLIMITED, DYNAMIC } default=DYNAMIC +## Only used if replay_throttling_policy.type == DYNAMIC: +## TODO consider just hardcoding values as they have never been tuned. +replay_throttling_policy.min_window_size int default=100 +replay_throttling_policy.max_window_size int default=10000 +replay_throttling_policy.window_size_increment int default=20 + +## Everything below are deprecated and ignored. Will go away at any time. + +## Deprecated and ignored, will soon go away +indexing.semiunboundtasklimit int default = 1000 + +## Include visits in the cache, if the visitoperation allows it. +## This will enable another separate cache of summary.cache.maxbytes size. +## IGNORED and DEPRECATED Will go away soon +summary.cache.allowvisitcaching bool default=true + +## Control number of cache entries preallocated. +## Default is no preallocation. +## Can be set to a higher number to avoid resizing. +## IGNORED and DEPRECATED Will go away soon +summary.cache.initialentries long default=0 restart + +## Skip crc32 check on read. +## IGNORED and DEPRECATED Will go away soon +summary.log.chunk.skipcrconread bool default=false + +## Overrides the number of threads used for writing fields across all document dbs. +## See feeding.concurrency for details. +## DEPRECATED - Remove usage +indexing.threads int default=1 restart + +## How much memory is set aside for caching. +## Now only used for caching of dictionary lookups. +## TODO Still relevant, check config model, seems unused. +index.cache.size long default=0 restart + ## Specifies which tensor implementation to use for all backend code. ## ## TENSOR_ENGINE (default) uses DefaultTensorEngine, which has been the production implementation for years. @@ -521,12 +543,6 @@ bucketdb.checksumtype enum {LEGACY, XXHASH64} default = LEGACY restart tensor_implementation enum {TENSOR_ENGINE, FAST_VALUE} default = FAST_VALUE ## Whether to report issues back to the container via protobuf field +## TODO Remove always on forward_issues bool default = true -## Chooses the throttling policy used to control the window size -## of the SharedOperationThrottler component used by the transaction log replay feed state. -replay_throttling_policy.type enum { UNLIMITED, DYNAMIC } default=DYNAMIC -## Only used if replay_throttling_policy.type == DYNAMIC: -replay_throttling_policy.min_window_size int default=100 -replay_throttling_policy.max_window_size int default=10000 -replay_throttling_policy.window_size_increment int default=20 diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp index 77cc1142f75..320e3f06d53 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp @@ -3,6 +3,7 @@ #include "attribute_manager_initializer.h" #include "attributes_initializer_base.h" #include "attribute_collection_spec_factory.h" +#include <vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h> #include <vespa/searchcorespi/index/i_thread_service.h> #include <future> @@ -22,12 +23,12 @@ class AttributeInitializerTask : public InitializerTask { private: AttributeInitializer::UP _initializer; - DocumentMetaStore::SP _documentMetaStore; + std::shared_ptr<DocumentMetaStoreAttribute> _documentMetaStore; InitializedAttributesResult &_result; public: AttributeInitializerTask(AttributeInitializer::UP initializer, - DocumentMetaStore::SP documentMetaStore, + std::shared_ptr<DocumentMetaStoreAttribute> documentMetaStore, InitializedAttributesResult &result) : _initializer(std::move(initializer)), _documentMetaStore(std::move(documentMetaStore)), @@ -52,14 +53,14 @@ class AttributeManagerInitializerTask : public vespalib::Executor::Task { std::promise<void> _promise; search::SerialNum _configSerialNum; - DocumentMetaStore::SP _documentMetaStore; + std::shared_ptr<DocumentMetaStoreAttribute> _documentMetaStore; AttributeManager::SP _attrMgr; InitializedAttributesResult &_attributesResult; public: AttributeManagerInitializerTask(std::promise<void> &&promise, search::SerialNum configSerialNum, - DocumentMetaStore::SP documentMetaStore, + std::shared_ptr<DocumentMetaStoreAttribute> documentMetaStore, AttributeManager::SP attrMgr, InitializedAttributesResult &attributesResult); ~AttributeManagerInitializerTask() override; @@ -69,7 +70,7 @@ public: AttributeManagerInitializerTask::AttributeManagerInitializerTask(std::promise<void> &&promise, search::SerialNum configSerialNum, - DocumentMetaStore::SP documentMetaStore, + std::shared_ptr<DocumentMetaStoreAttribute> documentMetaStore, AttributeManager::SP attrMgr, InitializedAttributesResult &attributesResult) : _promise(std::move(promise)), @@ -96,13 +97,13 @@ class AttributeInitializerTasksBuilder : public IAttributeInitializerRegistry private: InitializerTask &_attrMgrInitTask; InitializerTask::SP _documentMetaStoreInitTask; - DocumentMetaStore::SP _documentMetaStore; + std::shared_ptr<DocumentMetaStoreAttribute> _documentMetaStore; InitializedAttributesResult &_attributesResult; public: AttributeInitializerTasksBuilder(InitializerTask &attrMgrInitTask, InitializerTask::SP documentMetaStoreInitTask, - DocumentMetaStore::SP documentMetaStore, + std::shared_ptr<DocumentMetaStoreAttribute> documentMetaStore, InitializedAttributesResult &attributesResult); ~AttributeInitializerTasksBuilder() override; void add(AttributeInitializer::UP initializer) override; @@ -110,7 +111,7 @@ public: AttributeInitializerTasksBuilder::AttributeInitializerTasksBuilder(InitializerTask &attrMgrInitTask, InitializerTask::SP documentMetaStoreInitTask, - DocumentMetaStore::SP documentMetaStore, + std::shared_ptr<DocumentMetaStoreAttribute> documentMetaStore, InitializedAttributesResult &attributesResult) : _attrMgrInitTask(attrMgrInitTask), _documentMetaStoreInitTask(std::move(documentMetaStoreInitTask)), @@ -142,7 +143,7 @@ AttributeManagerInitializer::createAttributeSpec() const AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialNum, initializer::InitializerTask::SP documentMetaStoreInitTask, - DocumentMetaStore::SP documentMetaStore, + std::shared_ptr<DocumentMetaStoreAttribute> documentMetaStore, const AttributeManager & baseAttrMgr, const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h index 79203696ea1..a3a91b0d2e8 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h @@ -4,7 +4,6 @@ #include "attributemanager.h" #include "initialized_attributes_result.h" -#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> #include <vespa/searchcore/proton/common/alloc_strategy.h> #include <vespa/searchlib/common/serialnum.h> @@ -14,6 +13,8 @@ namespace searchcorespi::index { struct IThreadService; } namespace proton { +class DocumentMetaStoreAttribute; + /** * Class used to initialize an attribute manager. */ @@ -21,7 +22,7 @@ class AttributeManagerInitializer : public initializer::InitializerTask { private: search::SerialNum _configSerialNum; - DocumentMetaStore::SP _documentMetaStore; + std::shared_ptr<DocumentMetaStoreAttribute> _documentMetaStore; AttributeManager::SP _attrMgr; vespa::config::search::AttributesConfig _attrCfg; AllocStrategy _alloc_strategy; @@ -35,7 +36,7 @@ private: public: AttributeManagerInitializer(search::SerialNum configSerialNum, initializer::InitializerTask::SP documentMetaStoreInitTask, - DocumentMetaStore::SP documentMetaStore, + std::shared_ptr<DocumentMetaStoreAttribute> documentMetaStore, const AttributeManager & baseAttrMgr, const vespa::config::search::AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, diff --git a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp index 6a473df6689..ef9750b5f4c 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp @@ -5,6 +5,7 @@ #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/literalfieldvalue.h> #include <vespa/document/fieldvalue/predicatefieldvalue.h> +#include <vespa/document/fieldvalue/rawfieldvalue.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> @@ -22,6 +23,7 @@ #include <vespa/searchlib/attribute/changevector.hpp> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/searchlib/attribute/reference_attribute.h> +#include <vespa/searchlib/attribute/single_raw_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/classname.h> @@ -36,6 +38,7 @@ using vespalib::getClassName; using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using search::attribute::ReferenceAttribute; +using search::attribute::SingleRawAttribute; namespace { std::string toString(const FieldUpdate & update) { @@ -253,6 +256,26 @@ AttributeUpdater::handleUpdate(ReferenceAttribute &vec, uint32_t lid, const Valu } } +template <> +void +AttributeUpdater::handleUpdate(SingleRawAttribute& vec, uint32_t lid, const ValueUpdate& upd) +{ + LOG(spam, "handleUpdate(%s, %u): %s", vec.getName().c_str(), lid, toString(upd).c_str()); + ValueUpdate::ValueUpdateType op = upd.getType(); + assert(!vec.hasMultiValue()); + if (op == ValueUpdate::Assign) { + const AssignValueUpdate &assign(static_cast<const AssignValueUpdate &>(upd)); + if (assign.hasValue()) { + updateValue(vec, lid, assign.getValue()); + } + } else if (op == ValueUpdate::Clear) { + vec.clearDoc(lid); + } else { + LOG(warning, "Unsupported value update operation %s on singlevalue raw attribute %s", + upd.className(), vec.getName().c_str()); + } +} + void AttributeUpdater::handleUpdate(AttributeVector & vec, uint32_t lid, const FieldUpdate & fUpdate) { @@ -284,6 +307,8 @@ AttributeUpdater::handleUpdate(AttributeVector & vec, uint32_t lid, const FieldU handleUpdate(static_cast<TensorAttribute &>(vec), lid, vUp); } else if (vec.isReferenceType()) { handleUpdate(static_cast<ReferenceAttribute &>(vec), lid, vUp); + } else if (vec.is_raw_type()) { + handleUpdate(static_cast<SingleRawAttribute&>(vec), lid, vUp); } else { LOG(warning, "Unsupported attribute vector '%s' (classname=%s)", vec.getName().c_str(), getClassName(vec).c_str()); return; @@ -310,6 +335,9 @@ AttributeUpdater::handleValue(AttributeVector & vec, uint32_t lid, const FieldVa } else if (vec.isReferenceType()) { // ReferenceAttribute is never multivalue. updateValue(static_cast<ReferenceAttribute &>(vec), lid, val); + } else if (vec.is_raw_type()) { + // SingleRawAttribute is never multivalue + updateValue(static_cast<SingleRawAttribute&>(vec), lid, val); } else { LOG(warning, "Unsupported attribute vector '%s' (classname=%s)", vec.getName().c_str(), getClassName(vec).c_str()); return; @@ -499,6 +527,20 @@ AttributeUpdater::updateValue(ReferenceAttribute &vec, uint32_t lid, const Field } } +void +AttributeUpdater::updateValue(SingleRawAttribute& vec, uint32_t lid, const FieldValue& val) +{ + if (!val.isA(FieldValue::Type::RAW)) { + vec.clearDoc(lid); + throw UpdateException( + make_string("SingleRawAttribute must be updated with " + "RawFieldValue, but was '%s'", val.toString(false).c_str())); + } + const auto& raw_fv = static_cast<const RawFieldValue &>(val); + auto raw = raw_fv.getValueRef(); + vec.update(lid, {raw.data(), raw.size()}); +} + namespace { void diff --git a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h index 43b5f9e8477..338ec267e7a 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h +++ b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h @@ -14,7 +14,10 @@ namespace tensor { class PrepareResult; class TensorAttribute; } -namespace attribute {class ReferenceAttribute; } +namespace attribute { +class ReferenceAttribute; +class SingleRawAttribute; +} VESPA_DEFINE_EXCEPTION(UpdateException, vespalib::Exception); @@ -56,6 +59,7 @@ private: static void updateValue(PredicateAttribute & vec, uint32_t lid, const FieldValue & val); static void updateValue(tensor::TensorAttribute & vec, uint32_t lid, const FieldValue & val); static void updateValue(attribute::ReferenceAttribute & vec, uint32_t lid, const FieldValue & val); + static void updateValue(attribute::SingleRawAttribute& vec, uint32_t lid, const FieldValue& val); }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp index 16251ec7abf..8db8f00ed30 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp @@ -5,18 +5,14 @@ namespace proton { DocumentMetaStoreInitializerResult:: -DocumentMetaStoreInitializerResult(DocumentMetaStore::SP - documentMetaStore_in, - const search::TuneFileAttributes & - tuneFile_in) - : _documentMetaStore(documentMetaStore_in), +DocumentMetaStoreInitializerResult(std::shared_ptr<DocumentMetaStore> documentMetaStore_in, + const search::TuneFileAttributes & tuneFile_in) + : _documentMetaStore(std::move(documentMetaStore_in)), _tuneFile(tuneFile_in) { } -DocumentMetaStoreInitializerResult::~DocumentMetaStoreInitializerResult() -{ -} +DocumentMetaStoreInitializerResult::~DocumentMetaStoreInitializerResult() = default; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h index 22d59e0f8cd..5d43d2925cf 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h @@ -2,11 +2,11 @@ #pragma once -#include "documentmetastore.h" #include <vespa/searchlib/common/tunefileinfo.h> namespace proton { +class DocumentMetaStore; /** * The result after initializing document meta store component in a @@ -15,20 +15,18 @@ namespace proton { class DocumentMetaStoreInitializerResult { private: - DocumentMetaStore::SP _documentMetaStore; + std::shared_ptr<DocumentMetaStore> _documentMetaStore; const search::TuneFileAttributes _tuneFile; public: using SP = std::shared_ptr<DocumentMetaStoreInitializerResult>; - DocumentMetaStoreInitializerResult(DocumentMetaStore::SP - documentMetaStore_in, - const search::TuneFileAttributes & - tuneFile_in); + DocumentMetaStoreInitializerResult(std::shared_ptr<DocumentMetaStore> documentMetaStore_in, + const search::TuneFileAttributes & tuneFile_in); virtual ~DocumentMetaStoreInitializerResult(); - DocumentMetaStore::SP documentMetaStore() const { + std::shared_ptr<DocumentMetaStore> documentMetaStore() const { return _documentMetaStore; } const search::TuneFileAttributes &tuneFile() const { return _tuneFile; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 77452b60f21..27cffd67b86 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -228,10 +228,8 @@ DocumentMetaStore::onUpdateStat() { auto &compaction_strategy = getConfig().getCompactionStrategy(); vespalib::MemoryUsage usage = _metaDataStore.getMemoryUsage(); + usage.merge(_lidAlloc.getMemoryUsage()); usage.incAllocatedBytesOnHold(getGenerationHolder().get_held_bytes()); - size_t bvSize = _lidAlloc.getUsedLidsSize(); - usage.incAllocatedBytes(bvSize); - usage.incUsedBytes(bvSize); auto gid_to_lid_map_memory_usage = _gidToLidMap.getMemoryUsage(); _should_compact_gid_to_lid_map = compaction_strategy.should_compact_memory(gid_to_lid_map_memory_usage); usage.merge(gid_to_lid_map_memory_usage); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 72a062d85eb..e6b87ffdfc7 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -6,7 +6,6 @@ #include "documentmetastoreattribute.h" #include "lid_allocator.h" #include "lid_gid_key_comparator.h" -#include "lid_hold_list.h" #include "raw_document_meta_data.h" #include <vespa/searchcore/proton/common/subdbtype.h> #include <vespa/searchlib/queryeval/blueprint.h> diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp index 3be9282987c..57516f6d0aa 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp @@ -1,15 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentmetastorecontext.h" +#include "documentmetastore.h" namespace proton { -DocumentMetaStoreContext::ReadGuard::ReadGuard(const search::AttributeVector::SP &metaStoreAttr) : +DocumentMetaStoreContext::ReadGuard::ReadGuard(const std::shared_ptr<search::AttributeVector> & metaStoreAttr) : _guard(metaStoreAttr), _store(static_cast<const DocumentMetaStore &>(*_guard)) { } +const search::IDocumentMetaStore & +DocumentMetaStoreContext::ReadGuard::get() const { + return _store; +} + DocumentMetaStoreContext::DocumentMetaStoreContext(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB) : DocumentMetaStoreContext(std::move(bucketDB), DocumentMetaStore::getFixedName(), search::GrowStrategy()) {} @@ -23,8 +29,8 @@ DocumentMetaStoreContext::DocumentMetaStoreContext(std::shared_ptr<bucketdb::Buc } -DocumentMetaStoreContext::DocumentMetaStoreContext(const search::AttributeVector::SP &metaStoreAttr) : - _metaStoreAttr(metaStoreAttr), +DocumentMetaStoreContext::DocumentMetaStoreContext(std::shared_ptr<search::AttributeVector> metaStoreAttr) : + _metaStoreAttr(std::move(metaStoreAttr)), _metaStore(std::dynamic_pointer_cast<IDocumentMetaStore>(_metaStoreAttr)) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h index fce2968104d..f85515922f9 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h @@ -2,11 +2,13 @@ #pragma once -#include "documentmetastore.h" #include "i_document_meta_store_context.h" +#include <vespa/searchcommon/common/growstrategy.h> namespace proton { +class DocumentMetaStore; + /** * Class providing write and read interface to the document meta store. */ @@ -19,12 +21,12 @@ public: search::AttributeGuard _guard; const DocumentMetaStore &_store; public: - explicit ReadGuard(const search::AttributeVector::SP &metaStoreAttr); - const search::IDocumentMetaStore &get() const override { return _store; } + explicit ReadGuard(const std::shared_ptr<search::AttributeVector> &metaStoreAttr); + const search::IDocumentMetaStore &get() const override; }; private: - search::AttributeVector::SP _metaStoreAttr; - IDocumentMetaStore::SP _metaStore; + std::shared_ptr<search::AttributeVector> _metaStoreAttr; + std::shared_ptr<IDocumentMetaStore> _metaStore; public: explicit DocumentMetaStoreContext(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB); @@ -40,9 +42,9 @@ public: * Create a new context with the given document meta store encapsulated * as an attribute vector. */ - explicit DocumentMetaStoreContext(const search::AttributeVector::SP &metaStoreAttr); + explicit DocumentMetaStoreContext(std::shared_ptr<search::AttributeVector> metaStoreAttr); - proton::IDocumentMetaStore::SP getSP() const override { return _metaStore; } + std::shared_ptr<proton::IDocumentMetaStore> getSP() const override { return _metaStore; } proton::IDocumentMetaStore & get() override { return *_metaStore; } IReadGuard::UP getReadGuard() const override { return std::make_unique<ReadGuard>(_metaStoreAttr); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h index 5f97ba1e0b0..f3f343aef38 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h @@ -2,8 +2,10 @@ #pragma once +#include "lid_gid_key_comparator.h" +#include "i_store.h" +#include <vespa/vespalib/btree/btreeiterator.h> #include <vespa/searchlib/attribute/attributesaver.h> -#include "documentmetastore.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 5efc639064e..7b7d590d913 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -27,11 +27,19 @@ LidAllocator::LidAllocator(uint32_t size, _freeLids(size, capacity, genHolder, true, false), _usedLids(size, capacity, genHolder, false, true), _pendingHoldLids(size, capacity, genHolder, false, false), - _lidFreeListConstructed(false), _activeLids(size, capacity, genHolder, false, false), - _numActiveLids(0u) -{ + _numActiveLids(0u), + _lidFreeListConstructed(false) +{ } +vespalib::MemoryUsage +LidAllocator::getMemoryUsage() const { + vespalib::MemoryUsage usage; + size_t allocated = sizeof(*this) + _freeLids.byteSize() + _usedLids.byteSize() + + _pendingHoldLids.byteSize() + _activeLids.byteSize() + _holdLids.size(); + usage.incAllocatedBytes(allocated); + usage.incUsedBytes(allocated); + return usage; } LidAllocator::~LidAllocator() = default; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index 297218d17f8..2f38f47e4af 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -6,6 +6,7 @@ #include "lidstatevector.h" #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/vespalib/util/memoryusage.h> #include <atomic> namespace proton::documentmetastore { @@ -24,9 +25,9 @@ private: LidStateVector _freeLids; LidStateVector _usedLids; LidStateVector _pendingHoldLids; - bool _lidFreeListConstructed; LidStateVector _activeLids; std::atomic<uint32_t> _numActiveLids; + bool _lidFreeListConstructed; public: LidAllocator(uint32_t size, @@ -40,7 +41,7 @@ public: void registerLid(DocId lid) { _usedLids.setBit(lid); } void unregisterLid(DocId lid); void unregister_lids(const std::vector<DocId>& lids); - size_t getUsedLidsSize() const { return _usedLids.byteSize(); } + vespalib::MemoryUsage getMemoryUsage() const; void reclaim_memory(generation_t oldest_used_gen) { _holdLids.reclaim_memory(oldest_used_gen, _freeLids); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.cpp index ef0a244fc37..bacaf361d4a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.cpp @@ -14,7 +14,7 @@ LidHoldList::add(const uint32_t data, generation_t generation) { if (!_holdList.empty()) { assert(generation >= _holdList.back().second); } - _holdList.push_back(std::make_pair(data, generation)); + _holdList.emplace_back(data, generation); } void diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.h index 1000a11e6cb..0e80713d3c9 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.h @@ -37,6 +37,8 @@ public: **/ size_t size() const { return _holdList.size(); } + size_t byteSize() const { return _holdList.size() * sizeof(Element); } + /** * Clears the free list. **/ diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index f218193c02f..4798ebd8fbc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/util/singleexecutor.h> using vespalib::BlockingThreadStackExecutor; +using vespalib::ThreadStackExecutor; using vespalib::CpuUsage; using vespalib::SequencedTaskExecutor; using vespalib::SingleExecutor; @@ -21,12 +22,17 @@ namespace proton { namespace { std::unique_ptr<SyncableThreadExecutor> -createExecutorWithOneThread(uint32_t taskLimit, OptimizeFor optimize, - vespalib::Runnable::init_fun_t init_function) { - if (optimize == OptimizeFor::THROUGHPUT) { - return std::make_unique<SingleExecutor>(std::move(init_function), taskLimit); +createExecutorWithOneThread(const ThreadingServiceConfig & cfg, vespalib::Runnable::init_fun_t init_function) { + uint32_t taskLimit = cfg.defaultTaskLimit(); + if (cfg.optimize() == OptimizeFor::THROUGHPUT) { + uint32_t watermark = (cfg.kindOfwatermark() == 0) ? taskLimit / 10 : cfg.kindOfwatermark(); + return std::make_unique<SingleExecutor>(std::move(init_function), taskLimit, cfg.is_task_limit_hard(), watermark, 100ms); } else { - return std::make_unique<BlockingThreadStackExecutor>(1, taskLimit, std::move(init_function)); + if (cfg.is_task_limit_hard()) { + return std::make_unique<BlockingThreadStackExecutor>(1, taskLimit, std::move(init_function)); + } else { + return std::make_unique<ThreadStackExecutor>(1, std::move(init_function)); + } } } @@ -55,10 +61,8 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor & sharedEx _clock(clock), _masterExecutor(1, CpuUsage::wrap(master_executor, CpuUsage::Category::WRITE)), _master_task_limit(cfg.master_task_limit()), - _indexExecutor(createExecutorWithOneThread(cfg.defaultTaskLimit(), cfg.optimize(), - CpuUsage::wrap(index_executor, CpuUsage::Category::WRITE))), - _summaryExecutor(createExecutorWithOneThread(cfg.defaultTaskLimit(), cfg.optimize(), - CpuUsage::wrap(summary_executor, CpuUsage::Category::WRITE))), + _indexExecutor(createExecutorWithOneThread(cfg, CpuUsage::wrap(index_executor, CpuUsage::Category::WRITE))), + _summaryExecutor(createExecutorWithOneThread(cfg, CpuUsage::wrap(summary_executor, CpuUsage::Category::WRITE))), _masterService(_masterExecutor), _indexService(*_indexExecutor), _index_field_inverter(field_writer), diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index be0414421fe..de3ec9648d1 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1673,6 +1673,7 @@ ], "methods" : [ "public void <init>(com.yahoo.searchlib.rankingexpression.rule.ExpressionNode)", + "public com.yahoo.searchlib.rankingexpression.rule.ExpressionNode wrappedExpression()", "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index 75187d8ca19..7577c65527b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -209,6 +209,8 @@ public class TensorFunctionNode extends CompositeNode { this.expression = expression; } + public ExpressionNode wrappedExpression() { return expression; } + @Override public List<TensorFunction<Reference>> arguments() { if (expression instanceof CompositeNode) diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index f336742ca4b..7eb43faac18 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -6,15 +6,13 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> -#include <vespa/searchlib/attribute/multienumattribute.hpp> #include <vespa/searchlib/attribute/multistringattribute.h> -#include <vespa/searchlib/attribute/multivalueattribute.hpp> #include <vespa/searchlib/attribute/predicate_attribute.h> -#include <vespa/searchlib/attribute/singlenumericpostattribute.h> #include <vespa/searchlib/attribute/singlestringattribute.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> #include <vespa/searchlib/util/randomgenerator.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/update/arithmeticvalueupdate.h> @@ -49,6 +47,8 @@ string tmpDir("tmp"); string clsDir("clstmp"); string asuDir("asutmp"); +constexpr size_t sizeof_large_string_entry = sizeof(vespalib::datastore::UniqueStoreEntry<std::string>); + } namespace search { @@ -897,7 +897,7 @@ AttributeTest::testSingle() { AttributePtr ptr = createAttribute("sv-int32", Config(BasicType::INT32, CollectionType::SINGLE)); ptr->updateStat(true); - EXPECT_EQ(12288u, ptr->getStatus().getAllocated()); + EXPECT_EQ(4224u, ptr->getStatus().getAllocated()); EXPECT_EQ(0u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values); @@ -912,7 +912,7 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(347084u, ptr->getStatus().getAllocated()); + EXPECT_EQ(339020u, ptr->getStatus().getAllocated()); EXPECT_EQ(101852u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values); @@ -924,7 +924,7 @@ AttributeTest::testSingle() { AttributePtr ptr = createAttribute("sv-float", Config(BasicType::FLOAT, CollectionType::SINGLE)); ptr->updateStat(true); - EXPECT_EQ(12288u, ptr->getStatus().getAllocated()); + EXPECT_EQ(4224u, ptr->getStatus().getAllocated()); EXPECT_EQ(0u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<FloatingPointAttribute, double, float>(ptr, values); @@ -934,7 +934,7 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-float", cfg); ptr->updateStat(true); - EXPECT_EQ(347084u, ptr->getStatus().getAllocated()); + EXPECT_EQ(339020, ptr->getStatus().getAllocated()); EXPECT_EQ(101852u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<FloatingPointAttribute, double, float>(ptr, values); @@ -947,8 +947,8 @@ AttributeTest::testSingle() { AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE)); ptr->updateStat(true); - EXPECT_EQ(133216u, ptr->getStatus().getAllocated()); - EXPECT_EQ(53280u, ptr->getStatus().getUsed()); + EXPECT_EQ(117256u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(53240u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -957,8 +957,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-fs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(361584u, ptr->getStatus().getAllocated()); - EXPECT_EQ(105216u, ptr->getStatus().getUsed()); + EXPECT_EQ(345624u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(105176u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -1089,8 +1089,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(528304u, ptr->getStatus().getAllocated()); - EXPECT_EQ(512576u, ptr->getStatus().getUsed()); + EXPECT_EQ(512056u, ptr->getStatus().getAllocated()); + EXPECT_EQ(504392u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1099,8 +1099,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("flags", cfg); ptr->updateStat(true); - EXPECT_EQ(528304u, ptr->getStatus().getAllocated()); - EXPECT_EQ(512576u, ptr->getStatus().getUsed()); + EXPECT_EQ(512056u, ptr->getStatus().getAllocated()); + EXPECT_EQ(504392u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1109,8 +1109,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(885036u, ptr->getStatus().getAllocated()); - EXPECT_EQ(614448u, ptr->getStatus().getUsed()); + EXPECT_EQ(868788u, ptr->getStatus().getAllocated()); + EXPECT_EQ(606264u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1128,8 +1128,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-float", cfg); ptr->updateStat(true); - EXPECT_EQ(885036u, ptr->getStatus().getAllocated()); - EXPECT_EQ(614448u, ptr->getStatus().getUsed()); + EXPECT_EQ(868788u, ptr->getStatus().getAllocated()); + EXPECT_EQ(606264u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<FloatingPointAttribute, double>(ptr, values); } @@ -1140,8 +1140,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(649232u, ptr->getStatus().getAllocated()); - EXPECT_EQ(565856u, ptr->getStatus().getUsed()); + EXPECT_EQ(625088u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(557632u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } @@ -1150,8 +1150,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("afs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(899536u, ptr->getStatus().getAllocated()); - EXPECT_EQ(617812u, ptr->getStatus().getUsed()); + EXPECT_EQ(875392u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(609588u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 1d3995bc494..24213c53cbe 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -199,6 +199,8 @@ TEST("require that config can be converted") EXPECT_TRUE(assertDataType(AVBT::DOUBLE, CACAD::DOUBLE)); EXPECT_TRUE(assertDataType(AVBT::PREDICATE, CACAD::PREDICATE)); EXPECT_TRUE(assertDataType(AVBT::TENSOR, CACAD::TENSOR)); + EXPECT_TRUE(assertDataType(AVBT::REFERENCE, CACAD::REFERENCE)); + EXPECT_TRUE(assertDataType(AVBT::RAW, CACAD::RAW)); EXPECT_TRUE(assertDataType(AVBT::NONE, CACAD::NONE)); EXPECT_TRUE(assertCollectionType(AVCT::SINGLE, CACAC::SINGLE)); diff --git a/searchlib/src/tests/attribute/changevector/changevector_test.cpp b/searchlib/src/tests/attribute/changevector/changevector_test.cpp index c37d233217b..7820638ab3c 100644 --- a/searchlib/src/tests/attribute/changevector/changevector_test.cpp +++ b/searchlib/src/tests/attribute/changevector/changevector_test.cpp @@ -91,10 +91,10 @@ TEST("require that inserting empty vector does not affect the vector.") { TEST("require that we have control over buffer construction size") { CV a; EXPECT_EQUAL(0u, a.size()); - EXPECT_EQUAL(256u, a.capacity()); + EXPECT_EQUAL(4u, a.capacity()); a.clear(); EXPECT_EQUAL(0u, a.size()); - EXPECT_EQUAL(256u, a.capacity()); + EXPECT_EQUAL(4u, a.capacity()); } TEST("require that buffer can grow some") { diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 9043782e639..8464e0abfec 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -1022,7 +1022,7 @@ TEST_F("require that lid space can be increased after being compacted and then s TEST_F("require that there is control of static memory usage", Fixture) { vespalib::MemoryUsage usage = f.store.getMemoryUsage(); - EXPECT_EQUAL(584u + sizeof(std::mutex), sizeof(LogDataStore)); + EXPECT_EQUAL(536u + sizeof(LogDataStore::NameIdSet) + sizeof(std::mutex), sizeof(LogDataStore)); EXPECT_EQUAL(74108u, usage.allocatedBytes()); EXPECT_EQUAL(384u, usage.usedBytes()); } diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index a3f07959094..f3487f6e585 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -733,7 +733,6 @@ TYPED_TEST(HnswIndexTest, hnsw_graph_is_compacted) this->commit(); this->index->update_stat(compaction_strategy); mem_2 = this->commit_and_update_stat(); - EXPECT_LE(mem_2.usedBytes(), mem_1.usedBytes()); if (mem_2.usedBytes() == mem_1.usedBytes()) { break; } diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.hpp b/searchlib/src/vespa/searchlib/attribute/changevector.hpp index a81c14aa85c..08e914d4a1a 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/changevector.hpp @@ -15,6 +15,7 @@ namespace { // This number is selected to be large enough to hold bursts between commits constexpr size_t NUM_ELEMS_TO_RESERVE = 200; +constexpr size_t NUM_ELEMS_TO_RESERVE_INITIAL = 4; } @@ -22,7 +23,7 @@ template <typename T> ChangeVectorT<T>::ChangeVectorT() : _v() { - _v.reserve(roundUp2inN<T>(NUM_ELEMS_TO_RESERVE)); + _v.reserve(roundUp2inN<T>(NUM_ELEMS_TO_RESERVE_INITIAL)); } template <typename T> diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index a2079a88ddf..a799abb34c7 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -32,6 +32,7 @@ getDataTypeMap() map[AttributesConfig::Attribute::Datatype::PREDICATE] = BasicType::PREDICATE; map[AttributesConfig::Attribute::Datatype::TENSOR] = BasicType::TENSOR; map[AttributesConfig::Attribute::Datatype::REFERENCE] = BasicType::REFERENCE; + map[AttributesConfig::Attribute::Datatype::RAW] = BasicType::RAW; map[AttributesConfig::Attribute::Datatype::NONE] = BasicType::NONE; return map; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h index d0baaaf91cd..fe7c873c277 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h @@ -34,11 +34,12 @@ public: bool addDoc(DocId &docId) override; vespalib::ConstArrayRef<char> get_raw(DocId docid) const override; void set_raw(DocId docid, vespalib::ConstArrayRef<char> raw); - void update(DocId docid, vespalib::ConstArrayRef<char> raw) { set_raw(docid, raw); } - void append(DocId docid, vespalib::ConstArrayRef<char> raw, int32_t weight) { + bool update(DocId docid, vespalib::ConstArrayRef<char> raw) { set_raw(docid, raw); return true; } + bool append(DocId docid, vespalib::ConstArrayRef<char> raw, int32_t weight) { (void) docid; (void) raw; (void) weight; + return false; } bool isUndefined(DocId docid) const override; uint32_t clearDoc(DocId docId) override; diff --git a/storage/src/tests/common/testnodestateupdater.cpp b/storage/src/tests/common/testnodestateupdater.cpp index eeecf66995a..227d5ee8512 100644 --- a/storage/src/tests/common/testnodestateupdater.cpp +++ b/storage/src/tests/common/testnodestateupdater.cpp @@ -7,7 +7,8 @@ namespace storage { TestNodeStateUpdater::TestNodeStateUpdater(const lib::NodeType& type) - : _reported(new lib::NodeState(type, lib::State::UP)), + : _mutex(), + _reported(new lib::NodeState(type, lib::State::UP)), _current(new lib::NodeState(type, lib::State::UP)), _clusterStateBundle(std::make_shared<const lib::ClusterStateBundle>(lib::ClusterState())), _listeners(), diff --git a/storage/src/tests/common/testnodestateupdater.h b/storage/src/tests/common/testnodestateupdater.h index 6737ff4ebeb..207e94ff054 100644 --- a/storage/src/tests/common/testnodestateupdater.h +++ b/storage/src/tests/common/testnodestateupdater.h @@ -9,6 +9,7 @@ #pragma once #include <vespa/storage/common/nodestateupdater.h> +#include <mutex> namespace storage::lib { class ClusterState; @@ -18,6 +19,7 @@ namespace storage { struct TestNodeStateUpdater : public NodeStateUpdater { + mutable std::mutex _mutex; lib::NodeState::CSP _reported; lib::NodeState::CSP _current; std::shared_ptr<const lib::ClusterStateBundle> _clusterStateBundle; @@ -29,13 +31,14 @@ public: explicit TestNodeStateUpdater(const lib::NodeType& type); ~TestNodeStateUpdater() override; - lib::NodeState::CSP getReportedNodeState() const override { return _reported; } + lib::NodeState::CSP getReportedNodeState() const override { std::lock_guard guard(_mutex); return _reported; } lib::NodeState::CSP getCurrentNodeState() const override { return _current; } std::shared_ptr<const lib::ClusterStateBundle> getClusterStateBundle() const override; void addStateListener(StateListener& s) override { _listeners.push_back(&s); } void removeStateListener(StateListener&) override {} Lock::SP grabStateChangeLock() override { return std::make_shared<Lock>(); } void setReportedNodeState(const lib::NodeState& state) override { + std::lock_guard guard(_mutex); _reported = std::make_shared<lib::NodeState>(state); } void immediately_send_get_node_state_replies() override { diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index c3b87278345..011e7e4a31d 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -2763,6 +2763,7 @@ "methods" : [ "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.util.List)", "public java.util.List arguments()", + "public java.util.List selectorFunctions()", "public com.yahoo.tensor.functions.Slice withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java index 95987d9b886..11996b6a23d 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java @@ -106,6 +106,8 @@ public class ReduceJoin<NAMETYPE extends Name> extends CompositeTensorFunction<N return false; if (b.type().dimensions().size() != commonDimensions.dimensions().size()) return false; + } else if (dimensions.size() != commonDimensions.dimensions().size()) { + return false; } else { for (TensorType.Dimension dimension : commonDimensions.dimensions()) { if (!dimensions.contains(dimension.name())) diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java index a9a6df6ed4d..87e24306031 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java @@ -11,6 +11,7 @@ import com.yahoo.tensor.evaluation.EvaluationContext; import com.yahoo.tensor.evaluation.Name; import com.yahoo.tensor.evaluation.TypeContext; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Objects; @@ -47,6 +48,14 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY @Override public List<TensorFunction<NAMETYPE>> arguments() { return List.of(argument); } + public List<TensorFunction<NAMETYPE>> selectorFunctions() { + var result = new ArrayList<TensorFunction<NAMETYPE>>(); + for (var dimVal : subspaceAddress) { + dimVal.index().ifPresent(fun -> fun.asTensorFunction().ifPresent(tf -> result.add(tf))); + } + return result; + } + @Override public Slice<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { if (arguments.size() != 1) diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/ReduceJoinTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/ReduceJoinTestCase.java new file mode 100644 index 00000000000..d073c60d993 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/ReduceJoinTestCase.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.functions; + +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.evaluation.Name; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author arnej + */ +public class ReduceJoinTestCase { + + @Test + public void testReduceJoinTriggers() { + var a = Tensor.from("tensor(x[3])", "[1,2,3]"); + var b = Tensor.from("tensor(x[3])", "[4,5,6]"); + var fa = new ConstantTensor<Name>(a); + var fb = new ConstantTensor<Name>(a); + var j = new Join<Name>(fa, fb, ScalarFunctions.add()); + var r = new Reduce<Name>(j, Reduce.Aggregator.sum, "x"); + var rj = new ReduceJoin<Name>(r, j); + assertTrue(rj.canOptimize(a, b)); + } + + @Test + public void testReduceJoinUnoptimized() { + var a = Tensor.from("tensor(x[3])", "[1,2,3]"); + var b = Tensor.from("tensor(y[3])", "[4,5,6]"); + var fa = new ConstantTensor<Name>(a); + var fb = new ConstantTensor<Name>(a); + var j = new Join<Name>(fa, fb, ScalarFunctions.add()); + var r = new Reduce<Name>(j, Reduce.Aggregator.sum, "x"); + var rj = new ReduceJoin<Name>(r, j); + assertFalse(rj.canOptimize(a, b)); + } +} diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index 8ee9eaeb4f7..ccc3ab88c31 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -210,12 +210,12 @@ INSTANTIATE_TEST_SUITE_P(NumberStoreFreeListsDisabledMultiTest, TEST_P(NumberStoreTest, control_static_sizes) { static constexpr size_t sizeof_deque = vespalib::datastore::DataStoreBase::sizeof_entry_ref_hold_list_deque; - EXPECT_EQ(416u + sizeof_deque, sizeof(store)); + EXPECT_EQ(408u + sizeof_deque, sizeof(store)); EXPECT_EQ(240u + sizeof_deque, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType)); - EXPECT_EQ(112u, sizeof(NumberStoreTest::ArrayStoreType::SmallBufferType)); + EXPECT_EQ(104u, sizeof(NumberStoreTest::ArrayStoreType::SmallBufferType)); MemoryUsage usage = store.getMemoryUsage(); - EXPECT_EQ(202144u, usage.allocatedBytes()); - EXPECT_EQ(197776u, usage.usedBytes()); + EXPECT_EQ(202120u, usage.allocatedBytes()); + EXPECT_EQ(197752u, usage.usedBytes()); } TEST_P(NumberStoreTest, add_and_get_small_arrays_of_trivial_type) diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index 7121bf5ce11..df347267c7e 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -665,7 +665,7 @@ TEST(DataStoreTest, can_reuse_active_buffer_as_primary_buffer) } TEST(DataStoreTest, control_static_sizes) { - EXPECT_EQ(96, sizeof(BufferTypeBase)); + EXPECT_EQ(88, sizeof(BufferTypeBase)); EXPECT_EQ(24, sizeof(FreeList)); EXPECT_EQ(56, sizeof(BufferFreeList)); EXPECT_EQ(1, sizeof(BufferState::State)); diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 0de44eb373e..5ccf9a8908c 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -471,7 +471,7 @@ TEST_F(DoubleTest, nan_is_handled) TEST_F(DoubleTest, control_memory_usage) { static constexpr size_t sizeof_deque = vespalib::datastore::DataStoreBase::sizeof_entry_ref_hold_list_deque; - EXPECT_EQ(376u + sizeof_deque, sizeof(store)); + EXPECT_EQ(368u + sizeof_deque, sizeof(store)); EXPECT_EQ(144u, sizeof(BufferState)); EXPECT_EQ(28740u, store.get_values_memory_usage().allocatedBytes()); EXPECT_EQ(24804u, store.get_values_memory_usage().usedBytes()); diff --git a/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp index 3b1d244eb13..23b2dd19a85 100644 --- a/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp +++ b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp @@ -15,7 +15,7 @@ VESPA_THREAD_STACK_TAG(sequenced_executor) TEST("test that all tasks are executed") { std::atomic<uint64_t> counter(0); - SingleExecutor executor(sequenced_executor, 10); + SingleExecutor executor(sequenced_executor, 10, true, 1, 100ms); for (uint64_t i(0); i < 10; i++) { executor.execute(makeLambdaTask([&counter] {counter++;})); diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.h b/vespalib/src/vespa/vespalib/datastore/buffer_type.h index 53436df432f..bedbb2c984e 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.h @@ -85,7 +85,7 @@ protected: ElemCount dead_elems; BufferCounts() : used_elems(0), dead_elems(0) {} BufferCounts(ElemCount used_elems_in, ElemCount dead_elems_in) - : used_elems(used_elems_in), dead_elems(dead_elems_in) + : used_elems(used_elems_in), dead_elems(dead_elems_in) {} }; @@ -99,7 +99,7 @@ protected: const std::atomic<ElemCount>* dead_ptr; Element() noexcept : used_ptr(nullptr), dead_ptr(nullptr) {} Element(const std::atomic<ElemCount>* used_ptr_in, const std::atomic<ElemCount>* dead_ptr_in) noexcept - : used_ptr(used_ptr_in), dead_ptr(dead_ptr_in) + : used_ptr(used_ptr_in), dead_ptr(dead_ptr_in) {} }; std::vector<Element> _counts; @@ -119,7 +119,6 @@ protected: // Number of arrays needed before allocating a new buffer instead of just resizing the first one uint32_t _numArraysForNewBuffer; float _allocGrowFactor; - uint32_t _activeBuffers; uint32_t _holdBuffers; size_t _holdUsedElems; // Number of used elements in all held buffers for this type. AggregatedBufferCounts _aggr_counts; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp index 3e2adf21619..1d3ba27d6bf 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp @@ -78,7 +78,7 @@ UniqueStoreSmallStringBufferType::get_memory_allocator() const } UniqueStoreExternalStringBufferType::UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) - : BufferType<UniqueStoreEntry<std::string>>(array_size, 2u, max_arrays, NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER, ALLOC_GROW_FACTOR), + : BufferType<UniqueStoreEntry<std::string>>(array_size, 0u, max_arrays, NUM_ARRAYS_FOR_NEW_UNIQUESTORE_BUFFER, ALLOC_GROW_FACTOR), _memory_allocator(std::move(memory_allocator)) { } diff --git a/vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/vespalib/src/vespa/vespalib/util/singleexecutor.cpp index fa1aef61167..533fd293f71 100644 --- a/vespalib/src/vespa/vespalib/util/singleexecutor.cpp +++ b/vespalib/src/vespa/vespalib/util/singleexecutor.cpp @@ -6,10 +6,6 @@ namespace vespalib { -SingleExecutor::SingleExecutor(init_fun_t func, uint32_t taskLimit) - : SingleExecutor(func, taskLimit, true, taskLimit/10, 100ms) -{ } - SingleExecutor::SingleExecutor(init_fun_t func, uint32_t reservedQueueSize, bool isQueueSizeHard, uint32_t watermark, duration reactionTime) : _watermarkRatio(watermark < reservedQueueSize ? double(watermark) / reservedQueueSize : 1.0), _taskLimit(vespalib::roundUp2inN(reservedQueueSize)), diff --git a/vespalib/src/vespa/vespalib/util/singleexecutor.h b/vespalib/src/vespa/vespalib/util/singleexecutor.h index e8b7b0c44ad..975202aa38b 100644 --- a/vespalib/src/vespa/vespalib/util/singleexecutor.h +++ b/vespalib/src/vespa/vespalib/util/singleexecutor.h @@ -22,7 +22,6 @@ namespace vespalib { */ class SingleExecutor final : public vespalib::SyncableThreadExecutor, vespalib::Runnable { public: - SingleExecutor(init_fun_t func, uint32_t reservedQueueSize); SingleExecutor(init_fun_t func, uint32_t reservedQueueSize, bool isQueueSizeHard, uint32_t watermark, duration reactionTime); ~SingleExecutor() override; Task::UP execute(Task::UP task) override; |