diff options
author | gjoranv <gjoranv@gmail.com> | 2019-05-07 15:44:37 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-05-07 15:44:37 +0200 |
commit | 8b0272c3104080d1f293e6a709208d2ea149fc03 (patch) | |
tree | 75c617d1e16d4059ad06d629cb4cf0684806843e /config-model | |
parent | d342e55a3cc82d742fcb8295c9a45606197267f5 (diff) |
Revert "Gjoranv/New metrics proxy"
Diffstat (limited to 'config-model')
13 files changed, 40 insertions, 840 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index 2c45e292d01..715718f8bb5 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -96,12 +96,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>metrics-proxy</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-disc</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index 2f41b172ab6..d3da9fe1f03 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -1,4 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + * + * + */ package com.yahoo.searchdefinition; import com.yahoo.searchlib.rankingexpression.Reference; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index e0956097c84..dd27dd176e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -207,7 +207,7 @@ public class Admin extends AbstractConfigProducer implements Serializable { var metricsProxyCluster = new MetricsProxyContainerCluster(this, "metrics", deployState); int index = 0; for (var host : hosts) { - var container = new MetricsProxyContainer(metricsProxyCluster, index++, deployState.isHosted()); + var container = new MetricsProxyContainer(metricsProxyCluster, index++); addAndInitializeService(deployState.getDeployLogger(), host, container); metricsProxyCluster.addContainer(container); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java deleted file mode 100644 index 2df4008214a..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; - -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.getDefaultMetricsConsumer; - -/** - * Helper class to generate config for metrics consumers. - * - * @author gjoranv - */ -class ConsumersConfigGenerator { - - /** - * @param userConsumers The consumers set up by the user in services.xml - * @return A list of consumer builders (a mapping from consumer to its metrics) - */ - static List<Consumer.Builder> generate(Map<String, MetricsConsumer> userConsumers) { - // Normally, the user given consumers should not contain VESPA_CONSUMER_ID, but it's allowed for some internally used applications. - var allConsumers = new LinkedHashMap<>(userConsumers); - allConsumers.put(VESPA_CONSUMER_ID, combineConsumers(getDefaultMetricsConsumer(), allConsumers.get(VESPA_CONSUMER_ID))); - - return allConsumers.values().stream() - .map(ConsumersConfigGenerator::toConsumerBuilder) - .collect(Collectors.toList()); - } - - /* - * Returns a new consumer that is a combination of the two given consumers - * (ignoring the id of the consumers' metric sets). - * If a metric with the same id exists in both consumers, output name and - * dimensions from the 'overriding' consumer is used, but dimensions from 'original' - * are added if they don't exist in 'overriding'. - */ - private static MetricsConsumer combineConsumers(MetricsConsumer original, MetricsConsumer overriding) { - if (overriding == null) return original; - return addMetrics(original, overriding.getMetrics()); - } - - private static MetricsConsumer addMetrics(MetricsConsumer original, Map<String, Metric> metrics) { - Map<String, Metric> combinedMetrics = new LinkedHashMap<>(original.getMetrics()); - metrics.forEach((name, newMetric) -> - combinedMetrics.put(name, combineMetrics(original.getMetrics().get(name), newMetric))); - - return new MetricsConsumer(original.getId(), - new MetricSet(original.getMetricSet().getId(), combinedMetrics.values())); - } - - private static Metric combineMetrics(@Nullable Metric original, Metric newMetric) { - return original != null ? newMetric.addDimensionsFrom(original) : newMetric; - } - - private static Consumer.Builder toConsumerBuilder(MetricsConsumer consumer) { - Consumer.Builder builder = new Consumer.Builder().name(consumer.getId()); - consumer.getMetrics().values().forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); - return builder; - } - - private static Consumer.Metric.Builder toConsumerMetricBuilder(Metric metric) { - Consumer.Metric.Builder builder = new Consumer.Metric.Builder().name(metric.name) - .outputname(metric.outputName) - .description(metric.description); - metric.dimensions.forEach((name, value) -> builder.dimension(toMetricDimensionBuilder(name, value))); - return builder; - } - - private static Consumer.Metric.Dimension.Builder toMetricDimensionBuilder(String name, String value) { - return new Consumer.Metric.Dimension.Builder() - .key(name) - .value(value); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 1a66fa72c93..742196c91d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -1,66 +1,22 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnector; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServices; -import ai.vespa.metricsproxy.service.VespaServicesConfig; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.provision.ClusterMembership; import com.yahoo.vespa.model.container.Container; -import java.util.LinkedHashMap; -import java.util.Map; - import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CANONICAL_FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_TYPE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME; /** * Container running a metrics proxy. * * @author gjoranv */ -public class MetricsProxyContainer extends Container implements - NodeDimensionsConfig.Producer, - RpcConnectorConfig.Producer, - VespaServicesConfig.Producer -{ - - static final class NodeDimensionNames { - static final String FLAVOR = "flavor"; - static final String CANONICAL_FLAVOR = "canonicalFlavor"; - static final String CLUSTER_TYPE = "clustertype"; - static final String CLUSTER_ID = "clusterid"; - } - - private final boolean isHostedVespa; +public class MetricsProxyContainer extends Container { - public MetricsProxyContainer(AbstractConfigProducer parent, int index, boolean isHostedVespa) { + public MetricsProxyContainer(AbstractConfigProducer parent, int index) { super(parent, "" + index, index); - this.isHostedVespa = isHostedVespa; setProp("clustertype", "admin"); setProp("index", String.valueOf(index)); - addNodeSpecificComponents(); - } - - private void addNodeSpecificComponents() { - addMetricsProxyComponent(NodeDimensions.class); - addMetricsProxyComponent(RpcConnector.class); - addMetricsProxyComponent(VespaServices.class); - } - - int metricsRpcPortOffset() { - return numHttpServerPorts; } @Override @@ -70,7 +26,7 @@ public class MetricsProxyContainer extends Container implements @Override public int getWantedPort() { - return 19092; + return 19092; // TODO: current metrics-proxy uses 19091 as rpc port, will now get 19093. } @Override @@ -78,15 +34,9 @@ public class MetricsProxyContainer extends Container implements return true; } - // Must have predictable ports for both http and rpc. - @Override - public boolean requiresConsecutivePorts() { - return true; - } - @Override public int getPortCount() { - return metricsRpcPortOffset() + 1; + return super.getPortCount() + 1; } @Override @@ -95,43 +45,4 @@ public class MetricsProxyContainer extends Container implements portsMeta.on(numHttpServerPorts).tag("rpc").tag("metrics"); } - @Override - public String[] getPortSuffixes() { - var suffixes = super.getPortSuffixes(); - suffixes[metricsRpcPortOffset()] = "rpc/metrics"; - return suffixes; - } - - @Override - public void getConfig(RpcConnectorConfig.Builder builder) { - builder.port(getRelativePort(metricsRpcPortOffset())); - } - - @Override - public void getConfig(VespaServicesConfig.Builder builder) { - builder.service.addAll(VespaServicesConfigGenerator.generate(getHostResource().getServices())); - } - - @Override - public void getConfig(NodeDimensionsConfig.Builder builder) { - Map<String, String> dimensions = new LinkedHashMap<>(); - if (isHostedVespa) { - getHostResource().getFlavor().ifPresent(flavor -> { - dimensions.put(FLAVOR, flavor.name()); - dimensions.put(CANONICAL_FLAVOR, flavor.canonicalName()); - }); - - getHostResource().primaryClusterMembership().map(ClusterMembership::cluster).ifPresent(cluster -> { - dimensions.put(CLUSTER_TYPE, cluster.type().name()); - dimensions.put(CLUSTER_ID, cluster.id().value()); - }); - - builder.dimensions(dimensions); - } - } - - private void addMetricsProxyComponent(Class<?> componentClass) { - addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index d9a969ab7c9..437df42d531 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -1,183 +1,23 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.core.VespaMetrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcServer; -import ai.vespa.metricsproxy.core.MonitoringConfig; -import ai.vespa.metricsproxy.service.SystemPollerProvider; -import ai.vespa.metricsproxy.service.ConfigSentinelClient; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; -import com.yahoo.vespa.model.admin.monitoring.Monitoring; -import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.container.ContainerCluster; -import java.nio.file.Path; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; - -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.INSTANCE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.ZONE; -import static com.yahoo.vespa.model.container.xml.BundleMapper.JarSuffix.JAR_WITH_DEPS; -import static com.yahoo.vespa.model.container.xml.BundleMapper.bundlePathFromName; - /** * Container cluster for metrics proxy containers. * * @author gjoranv */ -public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyContainer> implements - ApplicationDimensionsConfig.Producer, - ConsumersConfig.Producer, - MonitoringConfig.Producer -{ - public static final Logger log = Logger.getLogger(MetricsProxyContainerCluster.class.getName()); - - private static final String METRICS_PROXY_NAME = "metrics-proxy"; - private static final Path METRICS_PROXY_BUNDLE_FILE = bundlePathFromName(METRICS_PROXY_NAME, JAR_WITH_DEPS); - static final String METRICS_PROXY_BUNDLE_NAME = "com.yahoo.vespa." + METRICS_PROXY_NAME; - - static final String DEFAULT_NAME_IN_MONITORING_SYSTEM = "vespa"; - static final int DEFAULT_MONITORING_INTERVAL = 1; - - static final class AppDimensionNames { - static final String ZONE = "zone"; - static final String APPLICATION_ID = "applicationId"; // tenant.app.instance - static final String TENANT = "tenantName"; - static final String APPLICATION = "applicationName"; - static final String INSTANCE = "instanceName"; - static final String LEGACY_APPLICATION = "app"; // app.instance - } - - private final AbstractConfigProducer<?> parent; - private final ApplicationId applicationId; +public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyContainer> { public MetricsProxyContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) { super(parent, name, name, deployState); - this.parent = parent; - applicationId = deployState.getProperties().applicationId(); - setRpcServerEnabled(false); addDefaultHandlersExceptStatus(); - - addPlatformBundle(METRICS_PROXY_BUNDLE_FILE); - addClusterComponents(); - } - - private void addClusterComponents() { - addMetricsProxyComponent(ApplicationDimensions.class); - addMetricsProxyComponent(ConfigSentinelClient.class); - addMetricsProxyComponent(ExternalMetrics.class); - addMetricsProxyComponent(MetricsConsumers.class); - addMetricsProxyComponent(MetricsManager.class); - addMetricsProxyComponent(RpcServer.class); - addMetricsProxyComponent(SystemPollerProvider.class); - addMetricsProxyComponent(VespaMetrics.class); } @Override protected void doPrepare(DeployState deployState) { } - @Override - public void getConfig(MonitoringConfig.Builder builder) { - builder.systemName(getSystemName()) - .intervalMinutes(getIntervalMinutes()); - } - - @Override - public void getConfig(ConsumersConfig.Builder builder) { - builder.consumer.addAll(ConsumersConfigGenerator.generate(getUserMetricsConsumers())); - } - - @Override - public void getConfig(ApplicationDimensionsConfig.Builder builder) { - if (isHostedVespa()) { - builder.dimensions(applicationDimensions()); - } - } - - // Returns the metricConsumers from services.xml - private Map<String, MetricsConsumer> getUserMetricsConsumers() { - return getAdmin() - .map(this::consumersInAdmin) - .orElse(Collections.emptyMap()); - } - - private Map<String, MetricsConsumer> consumersInAdmin(Admin admin) { - Metrics metrics = admin.getUserMetrics(); - return metrics.getConsumers(); - } - - private Optional<Admin> getAdmin() { - if (parent != null) { - AbstractConfigProducerRoot r = parent.getRoot(); - if (r instanceof VespaModel) { - VespaModel model = (VespaModel) r; - return Optional.ofNullable(model.getAdmin()); - } - } - return Optional.empty(); - } - - private String getSystemName() { - Monitoring monitoring = getMonitoringService(); - if (monitoring != null && ! monitoring.getClustername().equals("")) - return monitoring.getClustername(); - return DEFAULT_NAME_IN_MONITORING_SYSTEM; - } - - private int getIntervalMinutes() { - Monitoring monitoring = getMonitoringService(); - if (monitoring != null && monitoring.getInterval() != null) { - return monitoring.getInterval(); - } - return DEFAULT_MONITORING_INTERVAL; - } - - private void addMetricsProxyComponent(Class<?> componentClass) { - addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); - } - - private Map<String, String> applicationDimensions() { - Map<String, String> dimensions = new LinkedHashMap<>(); - dimensions.put(ZONE, zoneString(getZone())); - dimensions.put(APPLICATION_ID, serializeWithDots(applicationId)); - dimensions.put(TENANT, applicationId.tenant().value()); - dimensions.put(APPLICATION, applicationId.application().value()); - dimensions.put(INSTANCE, applicationId.instance().value()); - dimensions.put(LEGACY_APPLICATION, applicationId.application().value() + "." + applicationId.instance().value()); - return dimensions; - } - - // ApplicationId uses ':' as separator. - private static String serializeWithDots(ApplicationId applicationId) { - return applicationId.serializedForm().replace(':', '.'); - } - - static String zoneString(Zone zone) { - return zone.environment().value() + "." + zone.region().value(); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java deleted file mode 100644 index 08a81fad0a3..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.vespa.model.Service; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author gjoranv - */ -public class VespaServicesConfigGenerator { - - public static List<VespaServicesConfig.Service.Builder> generate(List<Service> services) { - return services.stream() - .filter(VespaServicesConfigGenerator::doIncludeServiceMetrics) - .map(VespaServicesConfigGenerator::toServiceBuilder) - .collect(Collectors.toList()); - } - - private static boolean doIncludeServiceMetrics(Service s) { - return s.getStartupCommand() != null || s.getServiceType().equals("configserver") || s.getServiceType().equals("config-sentinel"); - } - - private static VespaServicesConfig.Service.Builder toServiceBuilder(Service service) { - VespaServicesConfig.Service.Builder builder = new VespaServicesConfig.Service.Builder() - .configId(service.getConfigId()) - .name(service.getServiceName()) - .port(service.getHealthPort()) - .healthport(service.getHealthPort()); - - service.getDefaultMetricDimensions().forEach((name, value) -> builder.dimension(toServiceDimensionBuilder(name, value))); - return builder; - } - - private static VespaServicesConfig.Service.Dimension.Builder toServiceDimensionBuilder(String name, String value) { - return new VespaServicesConfig.Service.Dimension.Builder() - .key(name) - .value(value); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index f2bd67cdd81..f449feb5fc1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -10,7 +10,6 @@ import com.yahoo.container.QrConfig; import com.yahoo.container.core.ContainerHttpConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -37,10 +36,6 @@ import static com.yahoo.container.QrConfig.Filedistributor; import static com.yahoo.container.QrConfig.Rpc; /** - * Note about components: In general, all components should belong to the cluster and not the container. However, - * components that need node specific config must be added at the container level, along with the node-specific - * parts of the config generation (getConfig). - * * @author gjoranv * @author Einar M R Rosenvinge * @author Tony Vaagenes @@ -112,15 +107,11 @@ public abstract class Container extends AbstractService implements return components; } - public final void addComponent(Component c) { + public void addComponent(Component c) { components.addComponent(c); } - public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) { - addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec))); - } - - public final void addHandler(Handler h) { + public void addHandler(Handler h) { handlers.addComponent(h); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 1f0c3d6f84e..ad087288799 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -200,7 +200,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addVipHandler(); } - public final void addDefaultHandlersExceptStatus() { + public void addDefaultHandlersExceptStatus() { addDefaultRootHandler(); addMetricStateHandler(); addApplicationStatusHandler(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java index 4c5c14dad87..11d467a61d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java @@ -15,20 +15,7 @@ import java.util.Optional; */ public class BundleMapper { - public enum JarSuffix { - JAR_WITH_DEPS("-jar-with-dependencies.jar"), - DEPLOY("-deploy.jar"); - - private final String suffix; - - JarSuffix(String suffix) { - this.suffix = suffix; - } - } - public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); - - public static final String searchAndDocprocBundle = "container-search-and-docproc"; private static final Map<String, String> bundleFromClass; @@ -47,10 +34,6 @@ public class BundleMapper { return LIBRARY_PATH.resolve(fileName); } - public static Path bundlePathFromName(String name, JarSuffix suffix) { - return Paths.get(Defaults.getDefaults().underVespaHome(LIBRARY_PATH + name + suffix.suffix)); - } - /** * TODO: This is a temporary hack to ensure that users can use our internal components without * specifying the bundle in which the components reside. Ideally, this information diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 59a72aedaaf..efd13e82361 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -1,186 +1,45 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import org.junit.Rule; +import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Test; -import org.junit.rules.ExpectedException; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.zoneString; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_INSTANCE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getConsumersConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; /** * @author gjoranv */ public class MetricsProxyContainerClusterTest { - private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); - private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); - private static int numSystemMetrics = systemMetricSet.getMetrics().size(); - private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); - private static int numMetricsForDefaultConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - - @Test - public void default_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { - ConsumersConfig consumersConfig = getConsumersConfig(servicesWithAdminOnly()); - assertEquals(consumersConfig.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(numMetricsForDefaultConsumer, consumersConfig.consumer(0).metric().size()); - } - - @Test - public void vespa_is_a_reserved_consumer_id() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='vespa'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'Vespa' is not allowed as metrics consumer id"); - getConsumersConfig(services); - } - - @Test - public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { - String services = String.join("\n", - "<services application-type='hosted-infrastructure'>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='Vespa'>", - " <metric id='custom.metric1'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig config = getConsumersConfig(services); - assertEquals(1, config.consumer().size()); - - // All default metrics are retained - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertEquals(numMetricsForDefaultConsumer + 1, vespaConsumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); - } - - @Test - public void consumer_id_is_case_insensitive() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='A'/>", - " <consumer id='a'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'a' is used as id for two metrics consumers"); - getConsumersConfig(services); - } - - @Test - public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-metrics-only'>", - " <metric id='custom.metric1'/>", - " <metric id='custom.metric2'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - - assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - Metric customMetric2 = new Metric("custom.metric2"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); - assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); - } - @Test - public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-vespa-set'>", - " <metric-set id='vespa'/>", - " <metric id='my.extra.metric'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); + public void one_metrics_proxy_container_is_added_to_every_node() { + var numberOfHosts = 4; + var tester = new VespaModelTester(); + tester.enableMetricsProxyContainer(true); + tester.addHosts(4); - Metric customMetric = new Metric("my.extra.metric"); - assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); - } + VespaModel model = tester.createModel(servicesXml(), true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - @Test - public void hosted_application_propagates_application_dimensions() { - VespaModel hostedModel = getHostedModel(servicesWithAdminOnly()); - ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel); + for (var host : model.getHostSystem().getHosts()) { + assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); + } - assertEquals(zoneString(Zone.defaultZone()), config.dimensions(AppDimensionNames.ZONE)); - assertEquals(MY_TENANT, config.dimensions(AppDimensionNames.TENANT)); - assertEquals(MY_APPLICATION, config.dimensions(AppDimensionNames.APPLICATION)); - assertEquals(MY_INSTANCE, config.dimensions(AppDimensionNames.INSTANCE)); - assertEquals(MY_TENANT + "." + MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(AppDimensionNames.APPLICATION_ID)); - assertEquals(MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(AppDimensionNames.LEGACY_APPLICATION)); } - - private String servicesWithAdminOnly() { - return String.join("\n", "<services>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " </admin>", - "</services>" - ); + private String servicesXml() { + return String.join("\n", "<?xml version='1.0' encoding='utf-8' ?>", + "<services>", + " <container version='1.0' id='foo'>", + " <nodes count='2'/>", + " </container>", + " <content id='my-content' version='1.0'>", + " <documents />", + " <nodes count='2'/>", + " </content>", + "</services>"); } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java deleted file mode 100644 index ff486c6a437..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames; -import com.yahoo.vespa.model.test.VespaModelTester; -import org.junit.Test; - -import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getRpcConnectorConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getVespaServicesConfig; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - */ -public class MetricsProxyContainerTest { - - @Test - public void one_metrics_proxy_container_is_added_to_every_node() { - var numberOfHosts = 4; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(numberOfHosts); - - VespaModel model = tester.createModel(servicesWithManyNodes(), true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - - for (var host : model.getHostSystem().getHosts()) { - assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); - - long metricsProxies = host.getServices().stream() - .filter(s -> s.getClass().equals(MetricsProxyContainer.class)) - .count(); - assertThat(metricsProxies, is(1L)); - } - } - - @Test - public void http_server_is_running_on_expected_port() { - VespaModel model = getModel(servicesWithContent()); - MetricsProxyContainer container = (MetricsProxyContainer)model.id2producer().get(CONTAINER_CONFIG_ID); - assertEquals(19092, container.getSearchPort()); - assertEquals(19092, container.getHealthPort()); - assertEquals("http", container.getPortSuffixes()[0]); - - assertTrue(container.getPortsMeta().getTagsAt(0).contains("http")); - assertTrue(container.getPortsMeta().getTagsAt(0).contains("state")); - } - - @Test - public void rpc_server_is_running_on_expected_port() { - VespaModel model = getModel(servicesWithContent()); - - MetricsProxyContainer container = (MetricsProxyContainer)model.id2producer().get(CONTAINER_CONFIG_ID); - - int rpcPort = container.metricsRpcPortOffset(); - assertTrue(container.getPortsMeta().getTagsAt(rpcPort).contains("rpc")); - assertTrue(container.getPortsMeta().getTagsAt(rpcPort).contains("metrics")); - - assertEquals("rpc/metrics", container.getPortSuffixes()[rpcPort]); - - RpcConnectorConfig config = getRpcConnectorConfig(model); - assertEquals(19094, config.port()); - } - - @Test - public void hosted_application_propagates_node_dimensions() { - String services = servicesWithContent(); - VespaModel hostedModel = getHostedModel(services); - NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel); - - assertEquals("content", config.dimensions(NodeDimensionNames.CLUSTER_TYPE)); - assertEquals("my-content", config.dimensions(NodeDimensionNames.CLUSTER_ID)); - assertEquals(MY_FLAVOR, config.dimensions(NodeDimensionNames.FLAVOR)); - assertEquals(MY_FLAVOR, config.dimensions(NodeDimensionNames.CANONICAL_FLAVOR)); - } - - - @Test - public void vespa_services_config_has_all_services() { - VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent()); - assertEquals(6, vespaServicesConfig.service().size()); - - for (var service : vespaServicesConfig.service()) { - if (service.configId().equals("admin/cluster-controllers/0")) { - assertEquals("Wrong service name", "container-clustercontroller", service.name()); - assertEquals(1, service.dimension().size()); - assertEquals("clustername", service.dimension(0).key()); - assertEquals("cluster-controllers", service.dimension(0).value()); - } - } - } - - @Test - public void vespa_services_config_has_service_dimensions() { - VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent()); - for (var service : vespaServicesConfig.service()) { - if (service.configId().equals("admin/cluster-controllers/0")) { - assertEquals(1, service.dimension().size()); - assertEquals("clustername", service.dimension(0).key()); - assertEquals("cluster-controllers", service.dimension(0).value()); - } - } - } - - - private String servicesWithManyNodes() { - return String.join("\n", - "<services>", - " <container version='1.0' id='foo'>", - " <nodes count='2'/>", - " </container>", - " <content id='my-content' version='1.0'>", - " <documents />", - " <nodes count='2'/>", - " </content>", - "</services>"); - } - - private String servicesWithContent() { - return String.join("\n", - "<services>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " </admin>", - " <content version='1.0' id='my-content'>", - " <documents />", - " <nodes count='1' flavor='" + MY_FLAVOR + "' />", - " </content>", - "</services>" - ); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java deleted file mode 100644 index 3e5c8a6ef0d..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provisioning.FlavorsConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.test.VespaModelTester; - -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static org.junit.Assert.assertEquals; - -/** - * @author gjoranv - */ -class MetricsProxyModelTester { - - static final String MY_TENANT = "mytenant"; - static final String MY_APPLICATION = "myapp"; - static final String MY_INSTANCE = "myinstance"; - static final String MY_FLAVOR = "myflavor"; - - // Used for all configs that are produced by the container, not the cluster. - static final String CONTAINER_CONFIG_ID = "admin/metrics/0"; - - static VespaModel getModel(String servicesXml) { - var numberOfHosts = 1; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(numberOfHosts); - tester.setHosted(false); - return tester.createModel(servicesXml, true); - } - - static VespaModel getHostedModel(String servicesXml) { - var numberOfHosts = 2; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(flavorFromString(MY_FLAVOR), numberOfHosts); - tester.setHosted(true); - tester.setApplicationId(MY_TENANT, MY_APPLICATION, MY_INSTANCE); - return tester.createModel(servicesXml, true); - } - - static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) { - for (ConsumersConfig.Consumer.Metric m : consumer.metric()) { - if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname())) - return true; - } - return false; - } - - static ConsumersConfig.Consumer getCustomConsumer(String servicesXml) { - ConsumersConfig config = getConsumersConfig(servicesXml); - assertEquals(2, config.consumer().size()); - for (ConsumersConfig.Consumer consumer : config.consumer()) { - if (! consumer.name().equals(VESPA_CONSUMER_ID)) - return consumer; - } - throw new RuntimeException("Two consumers with the reserved id - this cannot happen."); - } - - static ConsumersConfig getConsumersConfig(String servicesXml) { - return getConsumersConfig(getModel(servicesXml)); - } - - private static ConsumersConfig getConsumersConfig(VespaModel model) { - String configId = "admin/metrics"; - return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), configId)); - } - - static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) { - String configId = "admin/metrics"; - return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), configId)); - } - - static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model) { - return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - static VespaServicesConfig getVespaServicesConfig(String servicesXml) { - VespaModel model = getModel(servicesXml); - return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) { - return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - private static Flavor flavorFromString(String name) { - return new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder(). - name(name))); - } - -} |