// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.otel; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.Service; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import static com.yahoo.vespa.defaults.Defaults.getDefaults; /** * @author olaa */ public class OpenTelemetryConfigGenerator { private final boolean useTls; private final String ca_file; private final String cert_file; private final String key_file; private List statePorts = new ArrayList<>(); private final Zone zone; private final ApplicationId applicationId; OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId) { this.zone = zone; this.applicationId = applicationId; boolean isCd = true; boolean isPublic = true; if (zone != null) { isCd = zone.system().isCd(); isPublic = zone.system().isPublic(); this.useTls = true; } else { // for manual testing this.useTls = false; } if (isCd) { if (isPublic) { this.ca_file = "/opt/vespa/var/vespa/trust-store.pem"; this.cert_file = "/var/lib/sia/certs/vespa.external.cd.tenant.cert.pem"; this.key_file = "/var/lib/sia/keys/vespa.external.cd.tenant.key.pem"; } else { this.ca_file = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; this.cert_file = "/var/lib/sia/certs/vespa.vespa.cd.tenant.cert.pem"; this.key_file = "/var/lib/sia/keys/vespa.vespa.cd.tenant.key.pem"; } } else { if (isPublic) { this.ca_file = "/opt/vespa/var/vespa/trust-store.pem"; this.cert_file = "/var/lib/sia/certs/vespa.external.tenant.cert.pem"; this.key_file = "/var/lib/sia/keys/vespa.external.tenant.key.pem"; } else { this.ca_file = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; this.cert_file = "/var/lib/sia/certs/vespa.vespa.tenant.cert.pem"; this.key_file = "/var/lib/sia/keys/vespa.vespa.tenant.key.pem"; } } } String receiverName(int index) { return "prometheus_simple/s" + index; } private void addReceivers(JsonGenerator g) throws java.io.IOException { g.writeFieldName("receivers"); g.writeStartObject(); int counter = 0; for (var statePort : statePorts) { addReceiver(g, ++counter, statePort); } g.writeEndObject(); // receivers } private void addReceiver(JsonGenerator g, int index, StatePortInfo statePort) throws java.io.IOException { g.writeFieldName(receiverName(index)); g.writeStartObject(); g.writeStringField("collection_interval", "60s"); g.writeStringField("endpoint", statePort.hostName() + ":" + statePort.portNumber()); addUrlInfo(g); if (useTls) addTls(g); { g.writeFieldName("labels"); var dimVals = serviceAttributes(statePort.service()); // these will be tagged as dimension-name/label-value // attributes on all metrics from this /state/v1 port g.writeStartObject(); for (var entry : dimVals.entrySet()) { if (entry.getValue() != null) { g.writeStringField(entry.getKey(), entry.getValue()); } } g.writeEndObject(); } g.writeEndObject(); } private void addTls(JsonGenerator g) throws java.io.IOException { g.writeFieldName("tls"); g.writeStartObject(); g.writeStringField("ca_file", ca_file); g.writeStringField("cert_file", cert_file); g.writeBooleanField("insecure_skip_verify", true); g.writeStringField("key_file", key_file); g.writeEndObject(); // tls } private void addUrlInfo(JsonGenerator g) throws java.io.IOException { g.writeStringField("metrics_path", "/state/v1/metrics"); g.writeFieldName("params"); g.writeStartObject(); g.writeStringField("format", "prometheus"); g.writeEndObject(); } private void addExporters(JsonGenerator g) throws java.io.IOException { g.writeFieldName("exporters"); g.writeStartObject(); addFileExporter(g); g.writeEndObject(); // exporters } private void addFileExporter(JsonGenerator g) throws java.io.IOException { g.writeFieldName("file"); g.writeStartObject(); g.writeStringField("path", getDefaults().underVespaHome("logs/vespa/otel-test.json")); { g.writeFieldName("rotation"); g.writeStartObject(); g.writeNumberField("max_megabytes", 10); g.writeNumberField("max_days", 3); g.writeNumberField("max_backups", 1); g.writeEndObject(); // rotation } g.writeEndObject(); // file } private void addProcessors(JsonGenerator g) throws java.io.IOException { g.writeFieldName("processors"); g.writeStartObject(); addResourceProcessor(g); g.writeEndObject(); } private void addResourceProcessor(JsonGenerator g) throws java.io.IOException { g.writeFieldName("resource"); g.writeStartObject(); g.writeFieldName("attributes"); g.writeStartArray(); // common attributes for all metrics from all services; // which application and which cloud/system/zone/environment addAttributeInsert(g, PublicDimensions.ZONE, zoneAttr()); addAttributeInsert(g, PublicDimensions.APPLICATION_ID, appIdAttr()); addAttributeInsert(g, "system", systemAttr()); addAttributeInsert(g, "tenantName", tenantAttr()); addAttributeInsert(g, "applicationName", appNameAttr()); addAttributeInsert(g, "instanceName", appInstanceAttr()); addAttributeInsert(g, "cloud", cloudAttr()); g.writeEndArray(); g.writeEndObject(); } private void addAttributeInsert(JsonGenerator g, String key, String value) throws java.io.IOException { if (value == null) return; g.writeStartObject(); g.writeStringField("key", key); g.writeStringField("value", value); g.writeStringField("action", "insert"); g.writeEndObject(); } private void addServiceBlock(JsonGenerator g) throws java.io.IOException { g.writeFieldName("service"); g.writeStartObject(); { g.writeFieldName("telemetry"); g.writeStartObject(); { g.writeFieldName("logs"); g.writeStartObject(); g.writeStringField("level", "debug"); g.writeEndObject(); } g.writeEndObject(); } { g.writeFieldName("pipelines"); g.writeStartObject(); addMetricsPipelines(g); g.writeEndObject(); // pipelines } g.writeEndObject(); // service } private void addMetricsPipelines(JsonGenerator g) throws java.io.IOException { g.writeFieldName("metrics"); g.writeStartObject(); { g.writeFieldName("receivers"); g.writeStartArray(); int counter = 0; for (var statePort : statePorts) { g.writeString(receiverName(++counter)); } g.writeEndArray(); } g.writeFieldName("processors"); g.writeStartArray(); g.writeString("resource"); g.writeEndArray(); { g.writeFieldName("exporters"); g.writeStartArray(); g.writeString("file"); g.writeEndArray(); } g.writeEndObject(); // metrics } // For now - mostly dummy config /* TODO: Create config 1. polling /state/v1 handler of every service (mostly done) 2. Processing with mapping/filtering from metric sets 3. Exporter to correct endpoint (alternatively amended) */ public String generate() { if (statePorts.isEmpty()) { return ""; } ByteArrayOutputStream out = new ByteArrayOutputStream(); try { JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8); g.writeStartObject(); addReceivers(g); addExporters(g); addProcessors(g); addServiceBlock(g); g.writeEndObject(); // root g.close(); } catch (java.io.IOException e) { System.err.println("unexpected error: " + e); return ""; } return out.toString(StandardCharsets.UTF_8); } void addStatePorts(List portList) { this.statePorts = portList; } List referencedPaths() { return List.of(ca_file, cert_file, key_file); } private String zoneAttr() { if (zone == null) return null; return zone.environment().value() + "." + zone.region().value(); } private String appIdAttr() { if (applicationId == null) return null; return applicationId.toFullString(); } private String systemAttr() { if (zone == null) return null; return zone.system().value(); } private String tenantAttr() { if (applicationId == null) return null; return applicationId.tenant().value(); } private String appNameAttr() { if (applicationId == null) return null; return applicationId.application().value(); } private String appInstanceAttr() { if (applicationId == null) return null; return applicationId.instance().value(); } private String cloudAttr() { if (zone == null) return null; return zone.cloud().name().value(); } private String getDeploymentCluster(ClusterSpec cluster) { if (applicationId == null) return null; if (zone == null) return null; String appString = applicationId.toFullString(); return String.join(".", appString, zone.environment().value(), zone.region().value(), cluster.id().value()); } private Map serviceAttributes(Service svc) { Map dimvals = new LinkedHashMap<>(); dimvals.put("instance", svc.getServiceName()); // should maybe be "local_service_name" ? dimvals.put("instanceType", svc.getServiceType()); // maybe "local_service_type", or remove String cName = svc.getServicePropertyString("clustername", null); if (cName != null) { // what about "clusterid" below, is it always the same? dimvals.put("clustername", cName); } String cType = svc.getServicePropertyString("clustertype", null); if (cType != null) { dimvals.put("clustertype", cType); } var hostResource = svc.getHost(); if (hostResource != null) { hostResource.spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> { dimvals.put(PublicDimensions.DEPLOYMENT_CLUSTER, getDeploymentCluster(cluster)); // overrides value above dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cluster.type().name()); // alternative to above dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cluster.id().value()); cluster.group().ifPresent(group -> dimvals.put(PublicDimensions.GROUP_ID, group.toString())); }); } return dimvals; } }