diff options
7 files changed, 81 insertions, 135 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java index 3f7ca7b46a7..9875cf434aa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.admin.otel; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -11,6 +12,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; @@ -145,6 +147,30 @@ public class OpenTelemetryConfigGenerator { g.writeFieldName("processors"); g.writeStartObject(); addResourceProcessor(g); + addRenameProcessor(g); + g.writeEndObject(); + } + private void addRenameProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("metricstransform/rename"); + g.writeStartObject(); + g.writeFieldName("transforms"); + g.writeStartArray(); + var metrics = MetricsConsumer.vespa9.metrics(); + for (var metric : metrics.values()) { + if (! metric.name.equals(metric.outputName)) { + String oldName = PrometheusUtil.sanitize(metric.name); + String newName = PrometheusUtil.sanitize(metric.outputName); + addRenameAction(g, oldName, newName); + } + } + g.writeEndArray(); + g.writeEndObject(); + } + private void addRenameAction(JsonGenerator g, String oldName, String newName) throws java.io.IOException { + g.writeStartObject(); + g.writeStringField("include", oldName); + g.writeStringField("action", "update"); + g.writeStringField("new_name", newName); g.writeEndObject(); } private void addResourceProcessor(JsonGenerator g) throws java.io.IOException { @@ -208,6 +234,7 @@ public class OpenTelemetryConfigGenerator { } g.writeFieldName("processors"); g.writeStartArray(); + g.writeString("metricstransform/rename"); g.writeString("resource"); g.writeEndArray(); { @@ -300,12 +327,13 @@ public class OpenTelemetryConfigGenerator { dimvals.put("instanceType", svc.getServiceType()); // maybe "local_service_type", or remove String cName = svc.getServicePropertyString("clustername", null); if (cName != null) { - // what about "clusterid" below, is it always the same? - dimvals.put("clustername", cName); + // overridden by cluster membership below (if available) + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cName); } String cType = svc.getServicePropertyString("clustertype", null); if (cType != null) { - dimvals.put("clustertype", cType); + // overridden by cluster membership below (if available) + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cType); } var hostResource = svc.getHost(); if (hostResource != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index fbac9e9d710..994837a2dbe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -237,24 +236,17 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> NodeSpec spec = getNextSearchNodeSpec(parentGroup); SearchNode searchNode; - TransactionLogServer tls; if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags(), syncTransactionLog); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); - - tls = new TransactionLogServer(searchNode, clusterName, syncTransactionLog); - tls.setHostResource(searchNode.getHostResource()); - tls.initService(deployState); } else { - searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, - tuning, resourceLimits, fractionOfMemoryReserved) + searchNode = new SearchNode.Builder("" + node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, + tuning, resourceLimits, fractionOfMemoryReserved, syncTransactionLog) .build(deployState, parent, element.getXml()); - tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } - searchNode.setTls(tls); if (searchCluster != null) { searchCluster.addSearcher(searchNode); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 61933c10504..08743290ae3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -55,17 +55,18 @@ public class SearchNode extends AbstractService implements private static final int UNUSED_2 = 2; private static final int UNUSED_3 = 3; private static final int HEALTH_PORT = 4; + private static final int TLS_PORT = 5; private final boolean isHostedVespa; private final boolean flushOnShutdown; private final NodeSpec nodeSpec; private final int distributionKey; private final String clusterName; - private TransactionLogServer tls; private final AbstractService serviceLayerService; private final Tuning tuning; private final ResourceLimits resourceLimits; private final double fractionOfMemoryReserved; + private final Boolean syncTransactionLog; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<SearchNode> { @@ -77,10 +78,11 @@ public class SearchNode extends AbstractService implements private final Tuning tuning; private final ResourceLimits resourceLimits; private final double fractionOfMemoryReserved; + private final Boolean syncTransactionLog; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Boolean syncTransactionLog) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -89,6 +91,7 @@ public class SearchNode extends AbstractService implements this.tuning = tuning; this.resourceLimits = resourceLimits; this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.syncTransactionLog = syncTransactionLog; } @Override @@ -96,7 +99,7 @@ public class SearchNode extends AbstractService implements Element producerSpec) { return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags(), syncTransactionLog); } } @@ -105,9 +108,9 @@ public class SearchNode extends AbstractService implements String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, boolean isHostedVespa, double fractionOfMemoryReserved, - ModelContext.FeatureFlags featureFlags) { + ModelContext.FeatureFlags featureFlags, Boolean syncTransactionLog) { SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, syncTransactionLog); if (featureFlags.loadCodeAsHugePages()) { node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true); } @@ -120,7 +123,7 @@ public class SearchNode extends AbstractService implements private SearchNode(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Boolean syncTransactionLog) { super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; @@ -134,9 +137,11 @@ public class SearchNode extends AbstractService implements portsMeta.on(UNUSED_2).tag("unused"); portsMeta.on(UNUSED_3).tag("unused"); portsMeta.on(HEALTH_PORT).tag("http").tag("json").tag("health").tag("state"); + portsMeta.on(TLS_PORT).tag("tls"); // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; + this.syncTransactionLog = syncTransactionLog; setPropertiesElastic(clusterName, distributionKey); addEnvironmentVariable("OMP_NUM_THREADS", 1); } @@ -172,6 +177,7 @@ public class SearchNode extends AbstractService implements from.allocatePort("unused/2"); from.allocatePort("unused/3"); from.allocatePort("health"); + from.allocatePort("tls"); } /** @@ -181,7 +187,7 @@ public class SearchNode extends AbstractService implements */ @Override public int getPortCount() { - return 5; + return 6; } /** @@ -198,6 +204,8 @@ public class SearchNode extends AbstractService implements return getHttpPort(); } + int getTlsPort() { return getRelativePort(TLS_PORT); } + @Override public String getServiceType() { return "searchnode"; @@ -219,7 +227,10 @@ public class SearchNode extends AbstractService implements builder.usefsync(false); } } - tls.getConfig(builder); + builder.listenport(getTlsPort()) + .basedir(getTlsDir()); + if (syncTransactionLog != null) + builder.usefsync(syncTransactionLog); } @Override @@ -227,14 +238,6 @@ public class SearchNode extends AbstractService implements return getHostName(); } - private TransactionLogServer getTransactionLogServer() { - return tls; - } - - public void setTls(TransactionLogServer tls) { - this.tls = tls; - } - public AbstractService getServiceLayerService() { return serviceLayerService; } @@ -260,7 +263,7 @@ public class SearchNode extends AbstractService implements httpport(getHttpPort()). clustername(getClusterName()). basedir(getBaseDir()). - tlsspec("tcp/" + getHost().getHostname() + ":" + getTransactionLogServer().getTlsPort()). + tlsspec("tcp/" + getHost().getHostname() + ":" + getTlsPort()). tlsconfigid(getConfigId()). slobrokconfigid(getClusterConfigId()). routingconfigid(getClusterConfigId()). @@ -305,6 +308,8 @@ public class SearchNode extends AbstractService implements new MetricsmanagerConfig.Consumer.Builder().name("log").tags("logdefault")); } + private String getTlsDir() { return "tls";} + @Override public Optional<String> getPreShutdownCommand() { if (flushOnShutdown) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java deleted file mode 100644 index 5617fd15cbc..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchlib.TranslogserverConfig; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.PortAllocBridge; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import org.w3c.dom.Element; - -/** - * @author hmusum - */ -public class TransactionLogServer extends AbstractService { - - private final Boolean useFsync; - - public TransactionLogServer(TreeConfigProducer<?> searchNode, String clusterName, Boolean useFsync) { - super(searchNode, "transactionlogserver"); - portsMeta.on(0).tag("tls"); - this.useFsync = useFsync; - setProp("clustername", clusterName); - setProp("clustertype", "search"); - } - - public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<TransactionLogServer> { - - private final String clusterName; - private final Boolean useFsync; - public Builder(String clusterName, Boolean useFsync) { - this.clusterName = clusterName; - this.useFsync = useFsync; - } - - @Override - protected TransactionLogServer doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { - return new TransactionLogServer(ancestor, clusterName, useFsync); - } - - } - - public int getPortCount() { - return 1; - } - - @Override - public void allocatePorts(int start, PortAllocBridge from) { - // NB: ignore "start" - from.allocatePort("tls"); - } - - /** - * Returns the port used by the TLS. - * - * @return The port. - */ - int getTlsPort() { - return getRelativePort(0); - } - - /** - * Returns the directory used by the TLS. - * - * @return The directory. - */ - private String getTlsDir() { - return "tls"; - } - - public void getConfig(TranslogserverConfig.Builder builder) { - builder.listenport(getTlsPort()) - .basedir(getTlsDir()); - if (useFsync != null) { - builder.usefsync(useFsync); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 924419daeae..3144748e7cc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -157,7 +157,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. assertEquals(1, c.getRoot().hostSystem().getHosts().size()); HostResource h = c.getRoot().hostSystem().getHost("mockhost"); - String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor", "transactionlogserver"}; + String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor"}; assertServices(h, expectedServices); assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); assertEquals("clu/search/cluster.clu/0", h.getService("searchnode").getConfigId()); @@ -205,7 +205,7 @@ public class ContentBuilderTest extends DomBuilderTest { HostResource h = cluster.getRoot().hostSystem().getHost("mockhost"); String [] expectedServices = { "logd", "configproxy", "config-sentinel", "configserver", "container", "logserver", - "slobrok", "storagenode", "distributor", "searchnode", "transactionlogserver", + "slobrok", "storagenode", "distributor", "searchnode", CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName }; assertServices(h, expectedServices); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index bc981c3de7c..499df4e5669 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -31,13 +30,8 @@ public class SearchNodeTest { assertEquals(expected, cfg.basedir()); } - private void prepare(MockRoot root, SearchNode node, Boolean useFsync) { + private void prepare(MockRoot root, SearchNode node) { Host host = new Host(root, "mockhost"); - TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); - tls.setHostResource(new HostResource(host)); - tls.setBasePort(100); - tls.initService(root.getDeployState()); - node.setTls(tls); node.setHostResource(new HostResource(host)); node.setBasePort(200); node.initService(root.getDeployState()); @@ -45,13 +39,15 @@ public class SearchNodeTest { } private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, - boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) { + boolean flushOnShutDown, boolean isHosted, + ModelContext.FeatureFlags featureFlags, + Boolean syncTransactionLog) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - null, null, isHosted, 0.0, featureFlags); + null, null, isHosted, 0.0, featureFlags, syncTransactionLog); } - private static SearchNode createSearchNode(MockRoot root) { - return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties()); + private static SearchNode createSearchNode(MockRoot root, Boolean syncTransactionLog) { + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties(), syncTransactionLog); } @Test @@ -64,15 +60,17 @@ public class SearchNodeTest { @Test void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); - prepare(root, node, true); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); + prepare(root, node); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @Test void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -81,7 +79,8 @@ public class SearchNodeTest { @Test void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); @@ -90,7 +89,8 @@ public class SearchNodeTest { private void verifyCodePlacement(boolean hugePages) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().loadCodeAsHugePages(hugePages)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().loadCodeAsHugePages(hugePages), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(hugePages, node.getEnvVars().get("VESPA_LOAD_CODE_AS_HUGEPAGES") != null); @@ -104,7 +104,8 @@ public class SearchNodeTest { private void verifySharedStringRepoReclaim(boolean sharedStringRepoNoReclaim) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(sharedStringRepoNoReclaim, node.getEnvVars().get("VESPA_SHARED_STRING_REPO_NO_RECLAIM") != null); @@ -120,10 +121,10 @@ public class SearchNodeTest { return new MockRoot("", new DeployState.Builder().properties(properties).build()); } - private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean useFsync) { + private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean syncTransactionLog) { MockRoot root = createRoot(properties); - SearchNode node = createSearchNode(root); - prepare(root, node, useFsync); + SearchNode node = createSearchNode(root, syncTransactionLog); + prepare(root, node); TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); node.getConfig(tlsBuilder); return tlsBuilder.build(); diff --git a/vespajlib/src/main/java/ai/vespa/json/Json.java b/vespajlib/src/main/java/ai/vespa/json/Json.java index 99ce9290be2..01bd5c2dc19 100644 --- a/vespajlib/src/main/java/ai/vespa/json/Json.java +++ b/vespajlib/src/main/java/ai/vespa/json/Json.java @@ -28,6 +28,7 @@ import java.util.stream.StreamSupport; import static com.yahoo.slime.Type.ARRAY; import static com.yahoo.slime.Type.STRING; +import static java.util.stream.Collectors.joining; /** * A {@link Slime} wrapper that throws {@link InvalidJsonException} on missing members or invalid types. @@ -165,8 +166,7 @@ public class Json implements Iterable<Json> { private void requirePresent() { if (isMissing()) throw createMissingMemberException(); } private InvalidJsonException createInvalidTypeException(Type... expected) { - var expectedTypesString = Arrays.stream(expected).map(this::toString) - .collect(java.util.stream.Collectors.joining("' or '", "'", "'")); + var expectedTypesString = Arrays.stream(expected).map(this::toString).collect(joining("' or '", "'", "'")); var pathString = path.isEmpty() ? "JSON" : "JSON member '%s'".formatted(path); return new InvalidJsonException( "Expected %s to be a %s but got '%s'" |