diff options
Diffstat (limited to 'config-model/src/main/java/com')
7 files changed, 58 insertions, 31 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index 4e7988a2006..b0f63ebb732 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -3,13 +3,16 @@ package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; import com.yahoo.schema.RankProfile; +import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import java.io.StringReader; import java.util.Set; /** @@ -83,7 +86,13 @@ public class InputRecorder extends ExpressionTransformer<RankProfileTransformCon throw new IllegalArgumentException("missing onnx model: " + arg); } for (String onnxInput : model.getInputMap().values()) { - neededInputs.add(onnxInput); + var reader = new StringReader(onnxInput); + try { + var asExpression = new RankingExpression(reader); + transform(asExpression.getRoot(), context); + } catch (ParseException e) { + throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage()); + } } return; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 12bce7a72a4..d69ddd1c5fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.model.PortAllocBridge; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; +import java.time.Duration; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -196,4 +197,6 @@ public class MetricsProxyContainer extends Container implements return ""; } + @Override public Optional<String> getPreShutdownCommand() { return Optional.of(prepareStopCommand(Duration.ofMinutes(6))); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index d83fd678af2..d6f0df8e051 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -10,7 +10,6 @@ import com.yahoo.metrics.Suffix; import java.util.Collections; import java.util.EnumSet; import java.util.LinkedHashSet; -import java.util.List; import java.util.Set; import static com.yahoo.metrics.Suffix.average; @@ -18,10 +17,10 @@ import static com.yahoo.metrics.Suffix.count; import static com.yahoo.metrics.Suffix.last; import static com.yahoo.metrics.Suffix.max; import static com.yahoo.metrics.Suffix.min; -import static com.yahoo.metrics.Suffix.sum; -import static com.yahoo.metrics.Suffix.rate; import static com.yahoo.metrics.Suffix.ninety_five_percentile; import static com.yahoo.metrics.Suffix.ninety_nine_percentile; +import static com.yahoo.metrics.Suffix.rate; +import static com.yahoo.metrics.Suffix.sum; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; import static java.util.Collections.singleton; @@ -93,6 +92,9 @@ public class VespaMetricSet { addMetric(metrics, "vds.server.network.server.insecure-connections-established"); addMetric(metrics, "vds.server.network.tls-connections-broken"); addMetric(metrics, "vds.server.network.failed-tls-config-reloads"); + // C++ capability metrics + addMetric(metrics, "vds.server.network.rpc-capability-checks-failed"); + addMetric(metrics, "vds.server.network.status-capability-checks-failed"); // C++ Fnet metrics addMetric(metrics, "vds.server.fnet.num-connections"); @@ -237,6 +239,9 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.SERVER_THREAD_POOL_SIZE, EnumSet.of(max, last)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. addMetric(metrics, ContainerMetrics.SERVER_ACTIVE_THREADS, EnumSet.of(min, max, sum, count, last)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. + addMetric(metrics, ContainerMetrics.JDISC_TLS_CAPABILITY_CHECKS_SUCCEEDED.rate()); + addMetric(metrics, ContainerMetrics.JDISC_TLS_CAPABILITY_CHECKS_FAILED.rate()); + return metrics; } @@ -332,6 +337,8 @@ public class VespaMetricSet { private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); @@ -407,19 +414,7 @@ public class VespaMetricSet { addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_ACCEPTED.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_UTILIZATION, EnumSet.of(max, sum, count)); - + // lid space addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 198f6b88798..04faff688f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -214,7 +214,10 @@ public class RankSetupValidator extends Validator { if (line.startsWith("debug\t")) continue; try { LogMessage logMessage = LogMessage.parseNativeFormat(line); - message.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n"); + message.append(logMessage.getLevel()) + .append(": ") + .append(logMessage.getPayload().replace("\\n", "\n\t")) + .append("\n"); } catch (InvalidLogFormatException e) { message.append(line).append("\n"); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 784902e2427..bc410670d5e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -12,11 +12,10 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.LogctlSpec; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.time.Duration; import java.util.List; import java.util.Optional; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - /** * A container that is typically used by container clusters set up from the user application. * @@ -95,11 +94,6 @@ public final class ApplicationContainer extends Container implements return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); } - @Override - public Optional<String> getPreShutdownCommand() { - int preshutdownTimeoutSeconds = 360; - int rpcTimeoutSeconds = preshutdownTimeoutSeconds + 10; - String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + preshutdownTimeoutSeconds; - return Optional.of(getDefaults().underVespaHome("bin/vespa-rpc-invoke") + " " + rpcParams); - } + @Override public Optional<String> getPreShutdownCommand() { return Optional.of(prepareStopCommand(Duration.ofMinutes(6))); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 6a6f4583eae..be9f3fa894f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.model.container.http.Http; import com.yahoo.vespa.model.container.http.JettyHttpServer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -38,6 +39,7 @@ import java.util.Optional; import static com.yahoo.container.QrConfig.Filedistributor; import static com.yahoo.container.QrConfig.Rpc; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; /** * Note about components: In general, all components should belong to the cluster and not the container. However, @@ -387,6 +389,12 @@ public abstract class Container extends AbstractService implements return dimensions; } + protected String prepareStopCommand(Duration timeout) { + long rpcTimeoutSeconds = timeout.toSeconds() + 10; + String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + timeout.toSeconds(); + return getDefaults().underVespaHome("bin/vespa-rpc-invoke") + " " + rpcParams; + } + private boolean messageBusEnabled() { return containerCluster().isPresent() && containerCluster().get().messageBusEnabled(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index 0abd7212017..76403d369dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.text.XML; import org.w3c.dom.Element; + +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -14,10 +17,22 @@ import java.util.stream.Collectors; */ public class ModelIdResolver { - private static final Map<String, String> providedModels = - Map.of("minilm-l6-v2", "https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx", - "mpnet-base-v2", "https://data.vespa.oath.cloud/onnx_models/sentence-all-mpnet-base-v2.onnx", - "bert-base-uncased", "https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt"); + private static Map<String, String> setupProvidedModels() { + Map<String, String> models = new HashMap<>(); + models.put("minilm-l6-v2", "https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx"); + models.put("mpnet-base-v2", "https://data.vespa.oath.cloud/onnx_models/sentence-all-mpnet-base-v2.onnx"); + models.put("bert-base-uncased", "https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt"); + models.put("flan-t5-vocab", "https://data.vespa.oath.cloud/onnx_models/flan-t5-spiece.model"); + models.put("flan-t5-small-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-small-encoder-model.onnx"); + models.put("flan-t5-small-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-small-decoder-model.onnx"); + models.put("flan-t5-base-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-base-encoder-model.onnx"); + models.put("flan-t5-base-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-base-decoder-model.onnx"); + models.put("flan-t5-large-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-large-encoder-model.onnx"); + models.put("flan-t5-large-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-large-decoder-model.onnx"); + return Collections.unmodifiableMap(models); + } + + private static final Map<String, String> providedModels = setupProvidedModels(); /** * Finds any config values of type 'model' below the given config element and |