diff options
134 files changed, 1348 insertions, 7213 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index 2c45e292d01..715718f8bb5 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -96,12 +96,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>metrics-proxy</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-disc</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index 2f41b172ab6..d3da9fe1f03 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -1,4 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + * + * + */ package com.yahoo.searchdefinition; import com.yahoo.searchlib.rankingexpression.Reference; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index e0956097c84..dd27dd176e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -207,7 +207,7 @@ public class Admin extends AbstractConfigProducer implements Serializable { var metricsProxyCluster = new MetricsProxyContainerCluster(this, "metrics", deployState); int index = 0; for (var host : hosts) { - var container = new MetricsProxyContainer(metricsProxyCluster, index++, deployState.isHosted()); + var container = new MetricsProxyContainer(metricsProxyCluster, index++); addAndInitializeService(deployState.getDeployLogger(), host, container); metricsProxyCluster.addContainer(container); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java deleted file mode 100644 index 2df4008214a..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; - -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.getDefaultMetricsConsumer; - -/** - * Helper class to generate config for metrics consumers. - * - * @author gjoranv - */ -class ConsumersConfigGenerator { - - /** - * @param userConsumers The consumers set up by the user in services.xml - * @return A list of consumer builders (a mapping from consumer to its metrics) - */ - static List<Consumer.Builder> generate(Map<String, MetricsConsumer> userConsumers) { - // Normally, the user given consumers should not contain VESPA_CONSUMER_ID, but it's allowed for some internally used applications. - var allConsumers = new LinkedHashMap<>(userConsumers); - allConsumers.put(VESPA_CONSUMER_ID, combineConsumers(getDefaultMetricsConsumer(), allConsumers.get(VESPA_CONSUMER_ID))); - - return allConsumers.values().stream() - .map(ConsumersConfigGenerator::toConsumerBuilder) - .collect(Collectors.toList()); - } - - /* - * Returns a new consumer that is a combination of the two given consumers - * (ignoring the id of the consumers' metric sets). - * If a metric with the same id exists in both consumers, output name and - * dimensions from the 'overriding' consumer is used, but dimensions from 'original' - * are added if they don't exist in 'overriding'. - */ - private static MetricsConsumer combineConsumers(MetricsConsumer original, MetricsConsumer overriding) { - if (overriding == null) return original; - return addMetrics(original, overriding.getMetrics()); - } - - private static MetricsConsumer addMetrics(MetricsConsumer original, Map<String, Metric> metrics) { - Map<String, Metric> combinedMetrics = new LinkedHashMap<>(original.getMetrics()); - metrics.forEach((name, newMetric) -> - combinedMetrics.put(name, combineMetrics(original.getMetrics().get(name), newMetric))); - - return new MetricsConsumer(original.getId(), - new MetricSet(original.getMetricSet().getId(), combinedMetrics.values())); - } - - private static Metric combineMetrics(@Nullable Metric original, Metric newMetric) { - return original != null ? newMetric.addDimensionsFrom(original) : newMetric; - } - - private static Consumer.Builder toConsumerBuilder(MetricsConsumer consumer) { - Consumer.Builder builder = new Consumer.Builder().name(consumer.getId()); - consumer.getMetrics().values().forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); - return builder; - } - - private static Consumer.Metric.Builder toConsumerMetricBuilder(Metric metric) { - Consumer.Metric.Builder builder = new Consumer.Metric.Builder().name(metric.name) - .outputname(metric.outputName) - .description(metric.description); - metric.dimensions.forEach((name, value) -> builder.dimension(toMetricDimensionBuilder(name, value))); - return builder; - } - - private static Consumer.Metric.Dimension.Builder toMetricDimensionBuilder(String name, String value) { - return new Consumer.Metric.Dimension.Builder() - .key(name) - .value(value); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 1a66fa72c93..742196c91d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -1,66 +1,22 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnector; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServices; -import ai.vespa.metricsproxy.service.VespaServicesConfig; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.provision.ClusterMembership; import com.yahoo.vespa.model.container.Container; -import java.util.LinkedHashMap; -import java.util.Map; - import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CANONICAL_FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_TYPE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME; /** * Container running a metrics proxy. * * @author gjoranv */ -public class MetricsProxyContainer extends Container implements - NodeDimensionsConfig.Producer, - RpcConnectorConfig.Producer, - VespaServicesConfig.Producer -{ - - static final class NodeDimensionNames { - static final String FLAVOR = "flavor"; - static final String CANONICAL_FLAVOR = "canonicalFlavor"; - static final String CLUSTER_TYPE = "clustertype"; - static final String CLUSTER_ID = "clusterid"; - } - - private final boolean isHostedVespa; +public class MetricsProxyContainer extends Container { - public MetricsProxyContainer(AbstractConfigProducer parent, int index, boolean isHostedVespa) { + public MetricsProxyContainer(AbstractConfigProducer parent, int index) { super(parent, "" + index, index); - this.isHostedVespa = isHostedVespa; setProp("clustertype", "admin"); setProp("index", String.valueOf(index)); - addNodeSpecificComponents(); - } - - private void addNodeSpecificComponents() { - addMetricsProxyComponent(NodeDimensions.class); - addMetricsProxyComponent(RpcConnector.class); - addMetricsProxyComponent(VespaServices.class); - } - - int metricsRpcPortOffset() { - return numHttpServerPorts; } @Override @@ -70,7 +26,7 @@ public class MetricsProxyContainer extends Container implements @Override public int getWantedPort() { - return 19092; + return 19092; // TODO: current metrics-proxy uses 19091 as rpc port, will now get 19093. } @Override @@ -78,15 +34,9 @@ public class MetricsProxyContainer extends Container implements return true; } - // Must have predictable ports for both http and rpc. - @Override - public boolean requiresConsecutivePorts() { - return true; - } - @Override public int getPortCount() { - return metricsRpcPortOffset() + 1; + return super.getPortCount() + 1; } @Override @@ -95,43 +45,4 @@ public class MetricsProxyContainer extends Container implements portsMeta.on(numHttpServerPorts).tag("rpc").tag("metrics"); } - @Override - public String[] getPortSuffixes() { - var suffixes = super.getPortSuffixes(); - suffixes[metricsRpcPortOffset()] = "rpc/metrics"; - return suffixes; - } - - @Override - public void getConfig(RpcConnectorConfig.Builder builder) { - builder.port(getRelativePort(metricsRpcPortOffset())); - } - - @Override - public void getConfig(VespaServicesConfig.Builder builder) { - builder.service.addAll(VespaServicesConfigGenerator.generate(getHostResource().getServices())); - } - - @Override - public void getConfig(NodeDimensionsConfig.Builder builder) { - Map<String, String> dimensions = new LinkedHashMap<>(); - if (isHostedVespa) { - getHostResource().getFlavor().ifPresent(flavor -> { - dimensions.put(FLAVOR, flavor.name()); - dimensions.put(CANONICAL_FLAVOR, flavor.canonicalName()); - }); - - getHostResource().primaryClusterMembership().map(ClusterMembership::cluster).ifPresent(cluster -> { - dimensions.put(CLUSTER_TYPE, cluster.type().name()); - dimensions.put(CLUSTER_ID, cluster.id().value()); - }); - - builder.dimensions(dimensions); - } - } - - private void addMetricsProxyComponent(Class<?> componentClass) { - addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index d9a969ab7c9..437df42d531 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -1,183 +1,23 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.core.VespaMetrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcServer; -import ai.vespa.metricsproxy.core.MonitoringConfig; -import ai.vespa.metricsproxy.service.SystemPollerProvider; -import ai.vespa.metricsproxy.service.ConfigSentinelClient; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; -import com.yahoo.vespa.model.admin.monitoring.Monitoring; -import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.container.ContainerCluster; -import java.nio.file.Path; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; - -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.INSTANCE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.ZONE; -import static com.yahoo.vespa.model.container.xml.BundleMapper.JarSuffix.JAR_WITH_DEPS; -import static com.yahoo.vespa.model.container.xml.BundleMapper.bundlePathFromName; - /** * Container cluster for metrics proxy containers. * * @author gjoranv */ -public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyContainer> implements - ApplicationDimensionsConfig.Producer, - ConsumersConfig.Producer, - MonitoringConfig.Producer -{ - public static final Logger log = Logger.getLogger(MetricsProxyContainerCluster.class.getName()); - - private static final String METRICS_PROXY_NAME = "metrics-proxy"; - private static final Path METRICS_PROXY_BUNDLE_FILE = bundlePathFromName(METRICS_PROXY_NAME, JAR_WITH_DEPS); - static final String METRICS_PROXY_BUNDLE_NAME = "com.yahoo.vespa." + METRICS_PROXY_NAME; - - static final String DEFAULT_NAME_IN_MONITORING_SYSTEM = "vespa"; - static final int DEFAULT_MONITORING_INTERVAL = 1; - - static final class AppDimensionNames { - static final String ZONE = "zone"; - static final String APPLICATION_ID = "applicationId"; // tenant.app.instance - static final String TENANT = "tenantName"; - static final String APPLICATION = "applicationName"; - static final String INSTANCE = "instanceName"; - static final String LEGACY_APPLICATION = "app"; // app.instance - } - - private final AbstractConfigProducer<?> parent; - private final ApplicationId applicationId; +public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyContainer> { public MetricsProxyContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) { super(parent, name, name, deployState); - this.parent = parent; - applicationId = deployState.getProperties().applicationId(); - setRpcServerEnabled(false); addDefaultHandlersExceptStatus(); - - addPlatformBundle(METRICS_PROXY_BUNDLE_FILE); - addClusterComponents(); - } - - private void addClusterComponents() { - addMetricsProxyComponent(ApplicationDimensions.class); - addMetricsProxyComponent(ConfigSentinelClient.class); - addMetricsProxyComponent(ExternalMetrics.class); - addMetricsProxyComponent(MetricsConsumers.class); - addMetricsProxyComponent(MetricsManager.class); - addMetricsProxyComponent(RpcServer.class); - addMetricsProxyComponent(SystemPollerProvider.class); - addMetricsProxyComponent(VespaMetrics.class); } @Override protected void doPrepare(DeployState deployState) { } - @Override - public void getConfig(MonitoringConfig.Builder builder) { - builder.systemName(getSystemName()) - .intervalMinutes(getIntervalMinutes()); - } - - @Override - public void getConfig(ConsumersConfig.Builder builder) { - builder.consumer.addAll(ConsumersConfigGenerator.generate(getUserMetricsConsumers())); - } - - @Override - public void getConfig(ApplicationDimensionsConfig.Builder builder) { - if (isHostedVespa()) { - builder.dimensions(applicationDimensions()); - } - } - - // Returns the metricConsumers from services.xml - private Map<String, MetricsConsumer> getUserMetricsConsumers() { - return getAdmin() - .map(this::consumersInAdmin) - .orElse(Collections.emptyMap()); - } - - private Map<String, MetricsConsumer> consumersInAdmin(Admin admin) { - Metrics metrics = admin.getUserMetrics(); - return metrics.getConsumers(); - } - - private Optional<Admin> getAdmin() { - if (parent != null) { - AbstractConfigProducerRoot r = parent.getRoot(); - if (r instanceof VespaModel) { - VespaModel model = (VespaModel) r; - return Optional.ofNullable(model.getAdmin()); - } - } - return Optional.empty(); - } - - private String getSystemName() { - Monitoring monitoring = getMonitoringService(); - if (monitoring != null && ! monitoring.getClustername().equals("")) - return monitoring.getClustername(); - return DEFAULT_NAME_IN_MONITORING_SYSTEM; - } - - private int getIntervalMinutes() { - Monitoring monitoring = getMonitoringService(); - if (monitoring != null && monitoring.getInterval() != null) { - return monitoring.getInterval(); - } - return DEFAULT_MONITORING_INTERVAL; - } - - private void addMetricsProxyComponent(Class<?> componentClass) { - addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); - } - - private Map<String, String> applicationDimensions() { - Map<String, String> dimensions = new LinkedHashMap<>(); - dimensions.put(ZONE, zoneString(getZone())); - dimensions.put(APPLICATION_ID, serializeWithDots(applicationId)); - dimensions.put(TENANT, applicationId.tenant().value()); - dimensions.put(APPLICATION, applicationId.application().value()); - dimensions.put(INSTANCE, applicationId.instance().value()); - dimensions.put(LEGACY_APPLICATION, applicationId.application().value() + "." + applicationId.instance().value()); - return dimensions; - } - - // ApplicationId uses ':' as separator. - private static String serializeWithDots(ApplicationId applicationId) { - return applicationId.serializedForm().replace(':', '.'); - } - - static String zoneString(Zone zone) { - return zone.environment().value() + "." + zone.region().value(); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java deleted file mode 100644 index 08a81fad0a3..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.vespa.model.Service; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author gjoranv - */ -public class VespaServicesConfigGenerator { - - public static List<VespaServicesConfig.Service.Builder> generate(List<Service> services) { - return services.stream() - .filter(VespaServicesConfigGenerator::doIncludeServiceMetrics) - .map(VespaServicesConfigGenerator::toServiceBuilder) - .collect(Collectors.toList()); - } - - private static boolean doIncludeServiceMetrics(Service s) { - return s.getStartupCommand() != null || s.getServiceType().equals("configserver") || s.getServiceType().equals("config-sentinel"); - } - - private static VespaServicesConfig.Service.Builder toServiceBuilder(Service service) { - VespaServicesConfig.Service.Builder builder = new VespaServicesConfig.Service.Builder() - .configId(service.getConfigId()) - .name(service.getServiceName()) - .port(service.getHealthPort()) - .healthport(service.getHealthPort()); - - service.getDefaultMetricDimensions().forEach((name, value) -> builder.dimension(toServiceDimensionBuilder(name, value))); - return builder; - } - - private static VespaServicesConfig.Service.Dimension.Builder toServiceDimensionBuilder(String name, String value) { - return new VespaServicesConfig.Service.Dimension.Builder() - .key(name) - .value(value); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index f2bd67cdd81..f449feb5fc1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -10,7 +10,6 @@ import com.yahoo.container.QrConfig; import com.yahoo.container.core.ContainerHttpConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -37,10 +36,6 @@ import static com.yahoo.container.QrConfig.Filedistributor; import static com.yahoo.container.QrConfig.Rpc; /** - * Note about components: In general, all components should belong to the cluster and not the container. However, - * components that need node specific config must be added at the container level, along with the node-specific - * parts of the config generation (getConfig). - * * @author gjoranv * @author Einar M R Rosenvinge * @author Tony Vaagenes @@ -112,15 +107,11 @@ public abstract class Container extends AbstractService implements return components; } - public final void addComponent(Component c) { + public void addComponent(Component c) { components.addComponent(c); } - public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) { - addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec))); - } - - public final void addHandler(Handler h) { + public void addHandler(Handler h) { handlers.addComponent(h); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 1f0c3d6f84e..ad087288799 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -200,7 +200,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addVipHandler(); } - public final void addDefaultHandlersExceptStatus() { + public void addDefaultHandlersExceptStatus() { addDefaultRootHandler(); addMetricStateHandler(); addApplicationStatusHandler(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java index 4c5c14dad87..11d467a61d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java @@ -15,20 +15,7 @@ import java.util.Optional; */ public class BundleMapper { - public enum JarSuffix { - JAR_WITH_DEPS("-jar-with-dependencies.jar"), - DEPLOY("-deploy.jar"); - - private final String suffix; - - JarSuffix(String suffix) { - this.suffix = suffix; - } - } - public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); - - public static final String searchAndDocprocBundle = "container-search-and-docproc"; private static final Map<String, String> bundleFromClass; @@ -47,10 +34,6 @@ public class BundleMapper { return LIBRARY_PATH.resolve(fileName); } - public static Path bundlePathFromName(String name, JarSuffix suffix) { - return Paths.get(Defaults.getDefaults().underVespaHome(LIBRARY_PATH + name + suffix.suffix)); - } - /** * TODO: This is a temporary hack to ensure that users can use our internal components without * specifying the bundle in which the components reside. Ideally, this information diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 59a72aedaaf..efd13e82361 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -1,186 +1,45 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import org.junit.Rule; +import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Test; -import org.junit.rules.ExpectedException; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.zoneString; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_INSTANCE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getConsumersConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; /** * @author gjoranv */ public class MetricsProxyContainerClusterTest { - private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); - private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); - private static int numSystemMetrics = systemMetricSet.getMetrics().size(); - private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); - private static int numMetricsForDefaultConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - - @Test - public void default_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { - ConsumersConfig consumersConfig = getConsumersConfig(servicesWithAdminOnly()); - assertEquals(consumersConfig.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(numMetricsForDefaultConsumer, consumersConfig.consumer(0).metric().size()); - } - - @Test - public void vespa_is_a_reserved_consumer_id() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='vespa'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'Vespa' is not allowed as metrics consumer id"); - getConsumersConfig(services); - } - - @Test - public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { - String services = String.join("\n", - "<services application-type='hosted-infrastructure'>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='Vespa'>", - " <metric id='custom.metric1'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig config = getConsumersConfig(services); - assertEquals(1, config.consumer().size()); - - // All default metrics are retained - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertEquals(numMetricsForDefaultConsumer + 1, vespaConsumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); - } - - @Test - public void consumer_id_is_case_insensitive() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='A'/>", - " <consumer id='a'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'a' is used as id for two metrics consumers"); - getConsumersConfig(services); - } - - @Test - public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-metrics-only'>", - " <metric id='custom.metric1'/>", - " <metric id='custom.metric2'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - - assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - Metric customMetric2 = new Metric("custom.metric2"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); - assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); - } - @Test - public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-vespa-set'>", - " <metric-set id='vespa'/>", - " <metric id='my.extra.metric'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); + public void one_metrics_proxy_container_is_added_to_every_node() { + var numberOfHosts = 4; + var tester = new VespaModelTester(); + tester.enableMetricsProxyContainer(true); + tester.addHosts(4); - Metric customMetric = new Metric("my.extra.metric"); - assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); - } + VespaModel model = tester.createModel(servicesXml(), true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - @Test - public void hosted_application_propagates_application_dimensions() { - VespaModel hostedModel = getHostedModel(servicesWithAdminOnly()); - ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel); + for (var host : model.getHostSystem().getHosts()) { + assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); + } - assertEquals(zoneString(Zone.defaultZone()), config.dimensions(AppDimensionNames.ZONE)); - assertEquals(MY_TENANT, config.dimensions(AppDimensionNames.TENANT)); - assertEquals(MY_APPLICATION, config.dimensions(AppDimensionNames.APPLICATION)); - assertEquals(MY_INSTANCE, config.dimensions(AppDimensionNames.INSTANCE)); - assertEquals(MY_TENANT + "." + MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(AppDimensionNames.APPLICATION_ID)); - assertEquals(MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(AppDimensionNames.LEGACY_APPLICATION)); } - - private String servicesWithAdminOnly() { - return String.join("\n", "<services>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " </admin>", - "</services>" - ); + private String servicesXml() { + return String.join("\n", "<?xml version='1.0' encoding='utf-8' ?>", + "<services>", + " <container version='1.0' id='foo'>", + " <nodes count='2'/>", + " </container>", + " <content id='my-content' version='1.0'>", + " <documents />", + " <nodes count='2'/>", + " </content>", + "</services>"); } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java deleted file mode 100644 index ff486c6a437..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames; -import com.yahoo.vespa.model.test.VespaModelTester; -import org.junit.Test; - -import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getRpcConnectorConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getVespaServicesConfig; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - */ -public class MetricsProxyContainerTest { - - @Test - public void one_metrics_proxy_container_is_added_to_every_node() { - var numberOfHosts = 4; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(numberOfHosts); - - VespaModel model = tester.createModel(servicesWithManyNodes(), true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - - for (var host : model.getHostSystem().getHosts()) { - assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); - - long metricsProxies = host.getServices().stream() - .filter(s -> s.getClass().equals(MetricsProxyContainer.class)) - .count(); - assertThat(metricsProxies, is(1L)); - } - } - - @Test - public void http_server_is_running_on_expected_port() { - VespaModel model = getModel(servicesWithContent()); - MetricsProxyContainer container = (MetricsProxyContainer)model.id2producer().get(CONTAINER_CONFIG_ID); - assertEquals(19092, container.getSearchPort()); - assertEquals(19092, container.getHealthPort()); - assertEquals("http", container.getPortSuffixes()[0]); - - assertTrue(container.getPortsMeta().getTagsAt(0).contains("http")); - assertTrue(container.getPortsMeta().getTagsAt(0).contains("state")); - } - - @Test - public void rpc_server_is_running_on_expected_port() { - VespaModel model = getModel(servicesWithContent()); - - MetricsProxyContainer container = (MetricsProxyContainer)model.id2producer().get(CONTAINER_CONFIG_ID); - - int rpcPort = container.metricsRpcPortOffset(); - assertTrue(container.getPortsMeta().getTagsAt(rpcPort).contains("rpc")); - assertTrue(container.getPortsMeta().getTagsAt(rpcPort).contains("metrics")); - - assertEquals("rpc/metrics", container.getPortSuffixes()[rpcPort]); - - RpcConnectorConfig config = getRpcConnectorConfig(model); - assertEquals(19094, config.port()); - } - - @Test - public void hosted_application_propagates_node_dimensions() { - String services = servicesWithContent(); - VespaModel hostedModel = getHostedModel(services); - NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel); - - assertEquals("content", config.dimensions(NodeDimensionNames.CLUSTER_TYPE)); - assertEquals("my-content", config.dimensions(NodeDimensionNames.CLUSTER_ID)); - assertEquals(MY_FLAVOR, config.dimensions(NodeDimensionNames.FLAVOR)); - assertEquals(MY_FLAVOR, config.dimensions(NodeDimensionNames.CANONICAL_FLAVOR)); - } - - - @Test - public void vespa_services_config_has_all_services() { - VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent()); - assertEquals(6, vespaServicesConfig.service().size()); - - for (var service : vespaServicesConfig.service()) { - if (service.configId().equals("admin/cluster-controllers/0")) { - assertEquals("Wrong service name", "container-clustercontroller", service.name()); - assertEquals(1, service.dimension().size()); - assertEquals("clustername", service.dimension(0).key()); - assertEquals("cluster-controllers", service.dimension(0).value()); - } - } - } - - @Test - public void vespa_services_config_has_service_dimensions() { - VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent()); - for (var service : vespaServicesConfig.service()) { - if (service.configId().equals("admin/cluster-controllers/0")) { - assertEquals(1, service.dimension().size()); - assertEquals("clustername", service.dimension(0).key()); - assertEquals("cluster-controllers", service.dimension(0).value()); - } - } - } - - - private String servicesWithManyNodes() { - return String.join("\n", - "<services>", - " <container version='1.0' id='foo'>", - " <nodes count='2'/>", - " </container>", - " <content id='my-content' version='1.0'>", - " <documents />", - " <nodes count='2'/>", - " </content>", - "</services>"); - } - - private String servicesWithContent() { - return String.join("\n", - "<services>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " </admin>", - " <content version='1.0' id='my-content'>", - " <documents />", - " <nodes count='1' flavor='" + MY_FLAVOR + "' />", - " </content>", - "</services>" - ); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java deleted file mode 100644 index 3e5c8a6ef0d..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provisioning.FlavorsConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.test.VespaModelTester; - -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static org.junit.Assert.assertEquals; - -/** - * @author gjoranv - */ -class MetricsProxyModelTester { - - static final String MY_TENANT = "mytenant"; - static final String MY_APPLICATION = "myapp"; - static final String MY_INSTANCE = "myinstance"; - static final String MY_FLAVOR = "myflavor"; - - // Used for all configs that are produced by the container, not the cluster. - static final String CONTAINER_CONFIG_ID = "admin/metrics/0"; - - static VespaModel getModel(String servicesXml) { - var numberOfHosts = 1; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(numberOfHosts); - tester.setHosted(false); - return tester.createModel(servicesXml, true); - } - - static VespaModel getHostedModel(String servicesXml) { - var numberOfHosts = 2; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(flavorFromString(MY_FLAVOR), numberOfHosts); - tester.setHosted(true); - tester.setApplicationId(MY_TENANT, MY_APPLICATION, MY_INSTANCE); - return tester.createModel(servicesXml, true); - } - - static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) { - for (ConsumersConfig.Consumer.Metric m : consumer.metric()) { - if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname())) - return true; - } - return false; - } - - static ConsumersConfig.Consumer getCustomConsumer(String servicesXml) { - ConsumersConfig config = getConsumersConfig(servicesXml); - assertEquals(2, config.consumer().size()); - for (ConsumersConfig.Consumer consumer : config.consumer()) { - if (! consumer.name().equals(VESPA_CONSUMER_ID)) - return consumer; - } - throw new RuntimeException("Two consumers with the reserved id - this cannot happen."); - } - - static ConsumersConfig getConsumersConfig(String servicesXml) { - return getConsumersConfig(getModel(servicesXml)); - } - - private static ConsumersConfig getConsumersConfig(VespaModel model) { - String configId = "admin/metrics"; - return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), configId)); - } - - static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) { - String configId = "admin/metrics"; - return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), configId)); - } - - static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model) { - return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - static VespaServicesConfig getVespaServicesConfig(String servicesXml) { - VespaModel model = getModel(servicesXml); - return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) { - return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - private static Flavor flavorFromString(String name) { - return new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder(). - name(name))); - } - -} diff --git a/configd/src/apps/sentinel/config-handler.cpp b/configd/src/apps/sentinel/config-handler.cpp index a434835c656..0f624da2bf0 100644 --- a/configd/src/apps/sentinel/config-handler.cpp +++ b/configd/src/apps/sentinel/config-handler.cpp @@ -155,6 +155,13 @@ ConfigHandler::doConfigure() } } _services.swap(services); + for (auto & entry : services) { + Service::UP svc = std::move(entry.second); + if (svc && svc->isRunning()) { + svc->remove(); + _orphans[entry.first] = std::move(svc); + } + } vespalib::ComponentConfigProducer::Config current("sentinel", _subscriber.getGeneration(), "ok"); _stateApi.myComponents.addConfig(current); } @@ -168,7 +175,7 @@ ConfigHandler::doWork() if (_subscriber.nextGeneration(0)) { doConfigure(); } - + handleRestarts(); handleCommands(); handleOutputs(); handleChildDeaths(); @@ -183,6 +190,16 @@ ConfigHandler::doWork() return false; } +void +ConfigHandler::handleRestarts() +{ + for (const auto & entry : _services) { + Service & svc = *(entry.second); + if (svc.wantsRestart()) { + svc.start(); + } + } +} void ConfigHandler::handleChildDeaths() @@ -198,6 +215,7 @@ ConfigHandler::handleChildDeaths() LOG(debug, "pid %d finished, Service:%s", (int)pid, service->name().c_str()); service->youExited(status); + _orphans.erase(service->name()); } else { LOG(warning, "Unknown child pid %d exited (wait-status = %d)", (int)pid, status); @@ -268,6 +286,12 @@ ConfigHandler::serviceByPid(pid_t pid) return service; } } + for (const auto & it : _orphans) { + Service *service = it.second.get(); + if (service->pid() == pid) { + return service; + } + } return nullptr; } diff --git a/configd/src/apps/sentinel/config-handler.h b/configd/src/apps/sentinel/config-handler.h index a1ae054f888..0af19a6363a 100644 --- a/configd/src/apps/sentinel/config-handler.h +++ b/configd/src/apps/sentinel/config-handler.h @@ -29,6 +29,7 @@ private: ConfigSubscriber _subscriber; ConfigHandle<SentinelConfig>::UP _sentinelHandle; ServiceMap _services; + ServiceMap _orphans; std::list<OutputConnection *> _outputConnections; CommandQueue _cmdQ; std::unique_ptr<RpcServer> _rpcServer; @@ -46,6 +47,7 @@ private: void handleCmd(const Cmd& cmd); void handleOutputs(); void handleChildDeaths(); + void handleRestarts(); static int listen(int port); void configure_port(int port); diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp index 20c2a6c9b00..941ea690024 100644 --- a/configd/src/apps/sentinel/service.cpp +++ b/configd/src/apps/sentinel/service.cpp @@ -72,11 +72,12 @@ Service::reconfigure(const SentinelConfig::Service& config) delete _config; _config = new SentinelConfig::Service(config); - if (_isAutomatic - && ((_state == READY) || (_state == FINISHED))) - { - LOG(debug, "%s: Restarting due to new config", name().c_str()); - start(); + if ((_state == READY) || (_state == FINISHED) || (_state == RESTARTING)) { + resetRestartPenalty(); + if (_isAutomatic) { + LOG(debug, "%s: Restarting due to new config", name().c_str()); + start(); + } } } @@ -148,19 +149,14 @@ Service::runCommand(const std::string & command) } } -int +void Service::start() { - // make sure the service does not restart in a tight loop: - time_t now = time(0); - int diff = now - _last_start; - if (diff < MAX_RESTART_PENALTY) { - incrementRestartPenalty(); + if (_state == REMOVING) { + LOG(warning, "tried to start '%s' in REMOVING state", name().c_str()); + return; } - if (diff > 10 * MAX_RESTART_PENALTY) { - resetRestartPenalty(); - } - now += _restartPenalty; // will delay start this much + time_t now = time(0); _last_start = now; // make a pipe, close the good ends of it, mark it close-on-exec @@ -181,7 +177,7 @@ Service::start() LOG(error, "%s: Attempted to start, but pipe() failed: %s", name().c_str(), strerror(errno)); setState(FAILED); - return -1; + return; } fflush(nullptr); @@ -196,7 +192,7 @@ Service::start() close(stdoutpipes[1]); close(stderrpipes[0]); close(stderrpipes[1]); - return -1; + return; } if (_pid == 0) { @@ -219,11 +215,6 @@ Service::start() if (stop()) { kill(getpid(), SIGTERM); } - if (_restartPenalty > 0) { - LOG(info, "%s: Applying %u sec restart penalty", name().c_str(), - _restartPenalty); - sleep(_restartPenalty); - } EV_STARTING(name().c_str()); runChild(pipes); // This function should not return. _exit(EXIT_FAILURE); @@ -260,8 +251,15 @@ Service::start() fcntl(stderrpipes[0], F_GETFL) | O_NONBLOCK); c = new OutputConnection(stderrpipes[0], p); _outputConnections.push_back(c); +} - return (_state == RUNNING) ? 0 : -1; +void +Service::remove() +{ + LOG(info, "%s: removed from config", name().c_str()); + setAutomatic(false); + terminate(false, false); + setState(REMOVING); } @@ -290,18 +288,24 @@ Service::youExited(int status) { // Someone did a waitpid() and figured out that we exited. _exitStatus = status; + bool expectedDeath = (_state == KILLING || _state == TERMINATING + || _state == REMOVING + || _state == KILLED || _state == TERMINATED); if (WIFEXITED(status)) { LOG(debug, "%s: Exited with exit code %d", name().c_str(), WEXITSTATUS(status)); EV_STOPPED(name().c_str(), _pid, WEXITSTATUS(status)); setState(FINISHED); } else if (WIFSIGNALED(status)) { - bool expectedDeath = (_state == KILLING || _state == TERMINATING - || _state == KILLED || _state == TERMINATED); if (expectedDeath) { EV_STOPPED(name().c_str(), _pid, WTERMSIG(status)); LOG(debug, "%s: Exited expectedly by signal %d", name().c_str(), WTERMSIG(status)); + if (_state == TERMINATING) { + setState(TERMINATED); + } else if (_state == KILLING) { + setState(KILLED); + } } else { EV_CRASH(name().c_str(), _pid, WTERMSIG(status)); setState(FAILED); @@ -316,18 +320,26 @@ Service::youExited(int status) _metrics.currentlyRunningServices--; _metrics.sentinel_running.sample(_metrics.currentlyRunningServices); - if (_state == TERMINATING) { - setState(TERMINATED); - } else if (_state == KILLING) { - setState(KILLED); + if (! expectedDeath) { + // make sure the service does not restart in a tight loop: + time_t now = time(0); + unsigned int diff = now - _last_start; + if (diff < MAX_RESTART_PENALTY) { + incrementRestartPenalty(); + } + if (diff > 10 * MAX_RESTART_PENALTY) { + resetRestartPenalty(); + } + if (diff < _restartPenalty) { + LOG(info, "%s: will delay start by %u seconds", name().c_str(), _restartPenalty - diff); + } } if (_isAutomatic && !stop()) { // ### Implement some rate limiting here maybe? LOG(debug, "%s: Restarting.", name().c_str()); - setState(READY); + setState(RESTARTING); _metrics.totalRestartsCounter++; _metrics.sentinel_restarts.add(); - start(); } } @@ -391,6 +403,8 @@ Service::isRunning() const case KILLED: case TERMINATED: case FAILED: + case RESTARTING: + case REMOVING: return false; case STARTING: @@ -402,6 +416,19 @@ Service::isRunning() const return true; // this will not be reached } +bool +Service::wantsRestart() const +{ + if (_state == RESTARTING) { + time_t now = time(0); + if (now > _last_start + _restartPenalty) { + return true; + } + } + return false; +} + + void Service::setAutomatic(bool autoStatus) { @@ -418,12 +445,17 @@ Service::incrementRestartPenalty() if (_restartPenalty > MAX_RESTART_PENALTY) { _restartPenalty = MAX_RESTART_PENALTY; } + LOG(info, "%s: incremented restart penalty to %u seconds", name().c_str(), _restartPenalty); } void Service::setState(ServiceState state) { + if (_state == REMOVING) { + // ignore further changes + return; + } if (state != _state) { LOG(debug, "%s: %s->%s", name().c_str(), stateName(_state), stateName(state)); _rawState = state; @@ -448,6 +480,8 @@ Service::stateName(ServiceState state) const case TERMINATED: return "TERMINATED"; case KILLED: return "KILLED"; case FAILED: return "FAILED"; + case RESTARTING: return "RESTARTING"; + case REMOVING: return "REMOVING"; } return "--BAD--"; } diff --git a/configd/src/apps/sentinel/service.h b/configd/src/apps/sentinel/service.h index 54bf1105a77..5a188f217ff 100644 --- a/configd/src/apps/sentinel/service.h +++ b/configd/src/apps/sentinel/service.h @@ -20,13 +20,14 @@ private: pid_t _pid; enum ServiceState { READY, STARTING, RUNNING, TERMINATING, KILLING, + RESTARTING, REMOVING, FINISHED, TERMINATED, KILLED, FAILED } _rawState; const enum ServiceState& _state; int _exitStatus; SentinelConfig::Service *_config; bool _isAutomatic; - static const int MAX_RESTART_PENALTY = 1800; + static const unsigned int MAX_RESTART_PENALTY = 1800; unsigned int _restartPenalty; time_t _last_start; @@ -55,11 +56,13 @@ public: int terminate() { return terminate(true, false); } - int start(); + void start(); + void remove(); void youExited(int status); // Call this if waitpid says it exited const vespalib::string & name() const; const char *stateName() const { return stateName(_state); } bool isRunning() const; + bool wantsRestart() const; int exitStatus() const { return _exitStatus; } const SentinelConfig::Service& serviceConfig() const { return *_config; } void setAutomatic(bool autoStatus); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3f2e2e4d36c..55f35de2443 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -53,6 +53,7 @@ import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; @@ -300,34 +301,35 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (tenant == null) return false; TenantApplications tenantApplications = tenant.getApplicationRepo(); - if (!tenantApplications.activeApplications().contains(applicationId)) return false; - - // Deleting an application is done by deleting the remote session and waiting - // until the config server where the deployment happened picks it up and deletes - // the local session - long sessionId = tenantApplications.requireActiveSessionOf(applicationId); - RemoteSession remoteSession = getRemoteSession(tenant, sessionId); - remoteSession.createDeleteTransaction().commit(); - - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); - // TODO: Add support for timeout in request - Duration waitTime = Duration.ofSeconds(60); - if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); - } else { - log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")"); - return false; - } + try (Lock lock = tenantApplications.lock(applicationId)) { + if ( ! tenantApplications.exists(applicationId)) return false; + // Deleting an application is done by deleting the remote session and waiting + // until the config server where the deployment happened picks it up and deletes + // the local session + long sessionId = tenantApplications.requireActiveSessionOf(applicationId); + RemoteSession remoteSession = getRemoteSession(tenant, sessionId); + remoteSession.createDeleteTransaction().commit(); + + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + // TODO: Add support for timeout in request + Duration waitTime = Duration.ofSeconds(60); + if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + } else { + log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")"); + return false; + } - NestedTransaction transaction = new NestedTransaction(); - transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested - // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server - // this is done asynchronously in application maintenance by the node repository) - transaction.add(tenantApplications.createDeleteTransaction(applicationId)); + NestedTransaction transaction = new NestedTransaction(); + transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested + // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server + // this is done asynchronously in application maintenance by the node repository) + transaction.add(tenantApplications.createDeleteTransaction(applicationId)); - hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); - transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); - transaction.commit(); + hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); + transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); + transaction.commit(); + } return true; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 1f301f06714..4be87253ceb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -11,14 +11,18 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.OptionalLong; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Logger; @@ -27,9 +31,9 @@ import java.util.stream.Collectors; /** * The applications of a tenant, backed by ZooKeeper. * - * Each application is stored under /config/v2/tenants/<tenant>/applications/<applications>, - * the root contains the currently active session, if any, and sub-paths /preparing contains the session id - * of whatever session may be activated next, if any, and /lock is used for synchronizing writes to all these paths. + * Each application is stored under /config/v2/tenants/<tenant>/applications/<application>, + * the root contains the currently active session, if any. Locks for synchronising writes to these paths, and changes + * to the config of this application, are found under /config/v2/tenants/<tenant>/locks/<application>. * * @author Ulf Lilleengen * @author jonmv @@ -40,16 +44,19 @@ public class TenantApplications { private final Curator curator; private final Path applicationsPath; + private final Path locksPath; // One thread pool for all instances of this class private static final ExecutorService pathChildrenExecutor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(TenantApplications.class.getName())); private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; + private final Map<ApplicationId, Lock> locks; - private TenantApplications(Curator curator, Path applicationsPath, ReloadHandler reloadHandler) { + private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { this.curator = curator; - this.applicationsPath = applicationsPath; - curator.create(applicationsPath); + this.applicationsPath = TenantRepository.getApplicationsPath(tenant); + this.locksPath = TenantRepository.getLocksPath(tenant); + this.locks = new ConcurrentHashMap<>(2); this.reloadHandler = reloadHandler; this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, pathChildrenExecutor); this.directoryCache.start(); @@ -57,7 +64,7 @@ public class TenantApplications { } public static TenantApplications create(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { - return new TenantApplications(curator, TenantRepository.getApplicationsPath(tenant), reloadHandler); + return new TenantApplications(curator, reloadHandler, tenant); } /** @@ -73,6 +80,10 @@ public class TenantApplications { .collect(Collectors.toUnmodifiableList()); } + public boolean exists(ApplicationId id) { + return curator.exists(applicationPath(id)); + } + /** Returns the id of the currently active session for the given application, if any. Throws on unknown applications. */ private OptionalLong activeSessionOf(ApplicationId id) { String data = curator.getData(applicationPath(id)).map(Utf8::toString) @@ -94,7 +105,9 @@ public class TenantApplications { * Creates a node for the given application, marking its existence. */ public void createApplication(ApplicationId id) { - curator.create(applicationPath(id)); + try (Lock lock = lock(id)) { + curator.create(applicationPath(id)); + } } /** @@ -113,7 +126,7 @@ public class TenantApplications { * Returns a transaction which deletes this application. */ public CuratorTransaction createDeleteTransaction(ApplicationId applicationId) { - return CuratorTransaction.from(CuratorOperations.delete(applicationPath(applicationId).getAbsolute()), curator); + return CuratorTransaction.from(CuratorOperations.deleteAll(applicationPath(applicationId).getAbsolute(), curator), curator); } /** @@ -130,6 +143,14 @@ public class TenantApplications { directoryCache.close(); } + /** Returns the lock for changing the session status of the given application. */ + public Lock lock(ApplicationId id) { + curator.create(lockPath(id)); + Lock lock = locks.computeIfAbsent(id, __ -> new Lock(lockPath(id).getAbsolute(), curator)); + lock.acquire(Duration.ofMinutes(1)); // These locks shouldn't be held for very long. + return lock; + } + private void childEvent(CuratorFramework client, PathChildrenCacheEvent event) { switch (event.getType()) { case CHILD_ADDED: @@ -163,4 +184,8 @@ public class TenantApplications { return applicationsPath.append(id.serializedForm()); } + private Path lockPath(ApplicationId id) { + return locksPath.append(id.serializedForm()); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 01bb4e2dc76..b6e1d1873c9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -119,9 +119,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { prepare(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - long sessionId = session.getSessionId(); - validateSessionStatus(session); - try (Lock lock = tenant.getSessionLock(timeout)) { + + try (Lock lock = tenant.getApplicationRepo().lock(session.getApplicationId())) { + validateSessionStatus(session); NestedTransaction transaction = new NestedTransaction(); transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(session.getApplicationId()), session, ignoreSessionStaleFailure)); @@ -129,13 +129,15 @@ public class Deployment implements com.yahoo.config.provision.Deployment { hostProvisioner.get().activate(transaction, session.getApplicationId(), session.getAllocatedHosts().getHosts()); } transaction.commit(); - session.waitUntilActivated(timeoutBudget); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new InternalServerException("Error activating application", e); } - log.log(LogLevel.INFO, session.logPre() + "Session " + sessionId + + + session.waitUntilActivated(timeoutBudget); + + log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + ( hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner" ) + ". Config generation " + session.getMetaData().getGeneration()); @@ -153,7 +155,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** Exposes the session of this for testing only */ public LocalSession session() { return session; } - + private long validateSessionStatus(LocalSession localSession) { long sessionId = localSession.getSessionId(); if (Session.Status.NEW.equals(localSession.getStatus())) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 8838daeb32e..26f437920ad 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -70,10 +70,7 @@ public class RemoteSession extends Session { } public synchronized ApplicationSet ensureApplicationLoaded() { - if (applicationSet == null) { - applicationSet = loadApplication(); - } - return applicationSet; + return applicationSet == null ? applicationSet = loadApplication() : applicationSet; } public Session.Status getStatus() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java index a68f4a396cd..88e71d7ddd1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java @@ -11,10 +11,8 @@ import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.RemoteSessionRepo; import com.yahoo.vespa.config.server.session.SessionFactory; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.Lock; import org.apache.zookeeper.data.Stat; -import java.time.Duration; import java.time.Instant; import java.util.Optional; @@ -30,7 +28,7 @@ public class Tenant implements TenantHandlerProvider { static final String SESSIONS = "sessions"; static final String APPLICATIONS = "applications"; - static final String SESSION_LOCK_PATH = "activateLock"; + static final String LOCKS = "locks"; private final TenantName name; private final RemoteSessionRepo remoteSessionRepo; @@ -38,7 +36,6 @@ public class Tenant implements TenantHandlerProvider { private final SessionFactory sessionFactory; private final LocalSessionRepo localSessionRepo; private final TenantApplications applicationRepo; - private final Lock sessionLock; private final RequestHandler requestHandler; private final ReloadHandler reloadHandler; private final TenantFileSystemDirs tenantFileSystemDirs; @@ -61,7 +58,6 @@ public class Tenant implements TenantHandlerProvider { this.remoteSessionRepo = remoteSessionRepo; this.sessionFactory = sessionFactory; this.localSessionRepo = localSessionRepo; - this.sessionLock = createLock(curator, path); this.applicationRepo = applicationRepo; this.tenantFileSystemDirs = tenantFileSystemDirs; this.curator = curator; @@ -110,14 +106,6 @@ public class Tenant implements TenantHandlerProvider { return localSessionRepo; } - /** - * This lock allows activation and deactivation of sessions under this tenant. - */ - public Lock getSessionLock(Duration timeout) { - sessionLock.acquire(timeout); - return sessionLock; - } - @Override public String toString() { return getName().value(); @@ -165,10 +153,4 @@ public class Tenant implements TenantHandlerProvider { curator.delete(path); } - private static Lock createLock(Curator curator, Path tenantPath) { - Path lockPath = tenantPath.append(SESSION_LOCK_PATH); - curator.create(lockPath); - return new Lock(lockPath.getAbsolute(), curator); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 9c74c9c1e67..53d01fdf933 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -62,6 +62,7 @@ public class TenantRepository { private static final TenantName DEFAULT_TENANT = TenantName.defaultName(); private static final Path tenantsPath = Path.fromString("/config/v2/tenants/"); + private static final Path locksPath = Path.fromString("/config/v2/locks/"); private static final Path vespaPath = Path.fromString("/vespa"); private static final Duration checkForRemovedApplicationsInterval = Duration.ofMinutes(1); private static final Logger log = Logger.getLogger(TenantRepository.class.getName()); @@ -106,6 +107,7 @@ public class TenantRepository { curator.framework().getConnectionStateListenable().addListener(this::stateChanged); curator.create(tenantsPath); + curator.create(locksPath); createSystemTenants(configserverConfig); curator.create(vespaPath); @@ -261,8 +263,10 @@ public class TenantRepository { * @param name name of the tenant */ private synchronized void writeTenantPath(TenantName name) { - Path tenantPath = getTenantPath(name); - curator.createAtomically(tenantPath, tenantPath.append(Tenant.SESSIONS), tenantPath.append(Tenant.APPLICATIONS)); + curator.createAtomically(TenantRepository.getTenantPath(name), + TenantRepository.getSessionsPath(name), + TenantRepository.getApplicationsPath(name), + TenantRepository.getLocksPath(name)); } /** @@ -406,4 +410,11 @@ public class TenantRepository { return getTenantPath(tenantName).append(Tenant.APPLICATIONS); } + /** + * Gets zookeeper path for locks for a tenant's applications. This is never cleaned, but shouldn't be a problem. + */ + public static Path getLocksPath(TenantName tenantName) { + return locksPath.append(tenantName.value()); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index fe34e6c361d..1430475e486 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -6,6 +6,7 @@ import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import com.yahoo.component.Version; @@ -31,6 +32,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.Lock; /** * A per tenant request handler, for handling reload (activate application) and getConfig requests for @@ -99,18 +101,31 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host */ @Override public void reloadConfig(ApplicationSet applicationSet) { - setLiveApp(applicationSet); - notifyReloadListeners(applicationSet); + ApplicationId id = applicationSet.getId(); + try (Lock lock = applications.lock(id)) { + if ( ! applications.exists(id)) + return; // Application was deleted before activation. + if (applicationSet.getApplicationGeneration() != applications.requireActiveSessionOf(id)) + return; // Application activated a new session before we got here. + + setLiveApp(applicationSet); + notifyReloadListeners(applicationSet); + } } @Override public void removeApplication(ApplicationId applicationId) { - if (applicationMapper.hasApplication(applicationId, clock.instant())) { - applicationMapper.remove(applicationId); - hostRegistry.removeHostsForKey(applicationId); - reloadListenersOnRemove(applicationId); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + try (Lock lock = applications.lock(applicationId)) { + if (applications.exists(applicationId)) + return; // Application was deployed again. + + if (applicationMapper.hasApplication(applicationId, clock.instant())) { + applicationMapper.remove(applicationId); + hostRegistry.removeHostsForKey(applicationId); + reloadListenersOnRemove(applicationId); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + } } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 4046384005d..9bd5c5f1614 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -81,6 +81,8 @@ public class TenantRepositoryTest { @Test public void testListenersAdded() throws IOException, SAXException { + tenantRepository.getTenant(tenant1).getApplicationRepo().createApplication(ApplicationId.defaultId()); + tenantRepository.getTenant(tenant1).getApplicationRepo().createPutTransaction(ApplicationId.defaultId(), 4).commit(); tenantRepository.getTenant(tenant1).getReloadHandler().reloadConfig(ApplicationSet.fromSingle( new Application(new VespaModel(MockApplicationPackage.createEmpty()), new ServerCache(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java index 96fee482092..6aa5aa7cd70 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java @@ -176,6 +176,8 @@ public class TenantRequestHandlerTest { public void testReloadConfig() throws IOException { ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); + server.applications().createApplication(applicationId); + server.applications().createPutTransaction(applicationId, 1).commit(); server.reloadConfig(reloadConfig(1)); assertThat(listener.reloaded.get(), is(1)); // Using only payload list for this simple test @@ -195,6 +197,7 @@ public class TenantRequestHandlerTest { listener.reloaded.set(0); feedApp(app2, 2, defaultApp(), true); + server.applications().createPutTransaction(applicationId, 2).commit(); server.reloadConfig(reloadConfig(2L)); configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); assertTrue(configResponse.isInternalRedeploy()); @@ -206,19 +209,34 @@ public class TenantRequestHandlerTest { @Test public void testRemoveApplication() { + ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default"); server.reloadConfig(reloadConfig(1)); + assertThat(listener.reloaded.get(), is(0)); + + server.applications().createApplication(appId); + server.applications().createPutTransaction(appId, 1).commit(); + server.reloadConfig(reloadConfig(1)); + assertThat(listener.reloaded.get(), is(1)); + + assertThat(listener.removed.get(), is(0)); + + server.removeApplication(appId); assertThat(listener.removed.get(), is(0)); - server.removeApplication(new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build()); + + server.applications().createDeleteTransaction(appId).commit(); + server.removeApplication(appId); assertThat(listener.removed.get(), is(1)); } @Test public void testResolveForAppId() { long id = 1L; - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(id))); ApplicationId appId = new ApplicationId.Builder() .tenant(tenant) .applicationName("myapp").instanceName("myinst").build(); + server.applications().createApplication(appId); + server.applications().createPutTransaction(appId, 1).commit(); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(id))); zkc.writeApplicationId(appId); RemoteSession session = new RemoteSession(appId.tenant(), id, componentRegistry, zkc); server.reloadConfig(session.ensureApplicationLoaded()); @@ -256,6 +274,8 @@ public class TenantRequestHandlerTest { } private void feedAndReloadApp(File appDir, long sessionId, ApplicationId appId) throws IOException { + server.applications().createApplication(appId); + server.applications().createPutTransaction(appId, sessionId).commit(); feedApp(appDir, sessionId, appId, false); SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); zkc.writeApplicationId(appId); @@ -291,9 +311,11 @@ public class TenantRequestHandlerTest { @Test public void testHasApplication() { assertdefaultAppNotFound(); + ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default"); + server.applications().createApplication(appId); + server.applications().createPutTransaction(appId, 1).commit(); server.reloadConfig(reloadConfig(1)); - assertTrue(server.hasApplication(new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(), - Optional.of(vespaVersion))); + assertTrue(server.hasApplication(appId, Optional.of(vespaVersion))); } private void assertdefaultAppNotFound() { @@ -302,10 +324,13 @@ public class TenantRequestHandlerTest { @Test public void testMultipleApplicationsReload() { + ApplicationId appId = ApplicationId.from(tenant.value(), "foo", "default"); assertdefaultAppNotFound(); + server.applications().createApplication(appId); + server.applications().createPutTransaction(appId, 1).commit(); server.reloadConfig(reloadConfig(1, "foo")); assertdefaultAppNotFound(); - assertTrue(server.hasApplication(new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(), + assertTrue(server.hasApplication(appId, Optional.of(vespaVersion))); assertThat(server.resolveApplicationId("doesnotexist"), is(ApplicationId.defaultId())); assertThat(server.resolveApplicationId("mytesthost"), is(new ApplicationId.Builder() @@ -318,6 +343,8 @@ public class TenantRequestHandlerTest { assertdefaultAppNotFound(); VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); + server.applications().createApplication(ApplicationId.defaultId()); + server.applications().createPutTransaction(ApplicationId.defaultId(), 1).commit(); server.reloadConfig(ApplicationSet.fromSingle(new Application(model, new ServerCache(), 1, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index c64d1f9f89b..f766e9331e9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -70,11 +70,21 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{*}", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/{*}"), - /** Path used to restart application nodes. */ // TODO move to the above when everyone is on new pipeline. - applicationRestart(Matcher.tenant, + /** Path used to restart development nodes. */ + developmentRestart(Matcher.tenant, Matcher.application, Optional.of("/api"), - "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/restart"), + "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{ignored}/restart", + "/application/v4/tenant/{tenant}/application/{application}/environment/perf/region/{region}/instance/{ignored}/restart"), + + /** Path used to restart production nodes. */ + productionRestart(Matcher.tenant, + Matcher.application, + Optional.of("/api"), + "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{ignored}/restart", + "/application/v4/tenant/{tenant}/application/{application}/environment/test/region/{region}/instance/{ignored}/restart", + "/application/v4/tenant/{tenant}/application/{application}/environment/staging/region/{region}/instance/{ignored}/restart"), + /** Paths used for development deployments. */ developmentDeployment(Matcher.tenant, Matcher.application, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 15745d69dc5..290382c6e6c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -80,12 +80,12 @@ enum Policy { /** Full access to application information and settings. */ applicationOperations(Privilege.grant(Action.write()) - .on(PathGroup.applicationInfo, PathGroup.applicationRestart) + .on(PathGroup.applicationInfo, PathGroup.productionRestart) .in(SystemName.all())), /** Full access to application development deployments. */ developmentDeployment(Privilege.grant(Action.all()) - .on(PathGroup.developmentDeployment) + .on(PathGroup.developmentDeployment, PathGroup.developmentRestart) .in(SystemName.all())), /** Full access to application production deployments. */ @@ -105,7 +105,7 @@ enum Policy { /** Full access to the additional tasks needed for continuous deployment. */ deploymentPipeline(Privilege.grant(Action.all()) // TODO remove when everyone is on new pipeline. - .on(PathGroup.buildService, PathGroup.applicationRestart) + .on(PathGroup.buildService, PathGroup.productionRestart) .in(SystemName.all())), /** Read access to all information in select systems. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 16c39a7e601..47fa73c3493 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -964,14 +964,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { * Deploy direct is when we want to redeploy the current application - retrieve version * info from the application package before deploying */ - if(deployDirectly && !applicationPackage.isPresent() && !applicationVersion.isPresent() && !vespaVersion.isPresent()) { + if(deployDirectly && applicationPackage.isEmpty() && applicationVersion.isEmpty() && vespaVersion.isEmpty()) { // Redeploy the existing deployment with the same versions. Optional<Deployment> deployment = controller.applications().get(applicationId) .map(Application::deployments) .flatMap(deployments -> Optional.ofNullable(deployments.get(zone))); - if(!deployment.isPresent()) + if(deployment.isEmpty()) throw new IllegalArgumentException("Can't redeploy application, no deployment currently exist"); ApplicationVersion version = deployment.get().applicationVersion(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 0612cee040c..99a0a8a4d4c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -161,6 +161,12 @@ public class Flags { "Takes effect on deployment through controller", APPLICATION_ID); + public static final UnboundStringFlag SILLY_ROUTING = defineStringFlag( + "silly-routing", "silly-default", + "Enables \"silly routing\" for the named application (in tenant:application:instance form)", + "Takes effect on next update to routing config", + HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index e76115931a3..ca812f92c49 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -3,7 +3,9 @@ package ai.vespa.hosted.api; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.SslContextBuilder; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -26,6 +28,8 @@ import java.time.Duration; import java.util.concurrent.Callable; import java.util.function.Supplier; +import static ai.vespa.hosted.api.Method.DELETE; +import static ai.vespa.hosted.api.Method.GET; import static ai.vespa.hosted.api.Method.POST; import static java.net.http.HttpRequest.BodyPublishers.ofByteArray; import static java.net.http.HttpRequest.BodyPublishers.ofInputStream; @@ -60,7 +64,7 @@ public abstract class ControllerHttpClient { return new MutualTlsControllerHttpClient(endpoint, privateKeyFile, certificateFile); } - /** Sends submission to the remote controller and returns the version of the accepted package, or throws if this fails. */ + /** Sends the given submission to the remote controller and returns the version of the accepted package, or throws if this fails. */ public String submit(Submission submission, TenantName tenant, ApplicationName application) { return toMessage(send(request(HttpRequest.newBuilder(applicationPath(tenant, application).resolve("submit")) .timeout(Duration.ofMinutes(30)), @@ -70,6 +74,37 @@ public abstract class ControllerHttpClient { .addFile("applicationTestZip", submission.applicationTestZip())))); } + /** Sends the given deployment to the given application in the given zone, or throws if this fails. */ + public DeploymentResult deploy(Deployment deployment, ApplicationId id, ZoneId zone) { + return toDeploymentResult(send(request(HttpRequest.newBuilder(deploymentPath(id, zone)) + .timeout(Duration.ofMinutes(60)), + POST, + toDataStream(deployment)))); + } + + /** Deactivates the deployment of the given application in the given zone. */ + public String deactivate(ApplicationId id, ZoneId zone) { + return asText(send(request(HttpRequest.newBuilder(deploymentPath(id, zone)) + .timeout(Duration.ofSeconds(10)), + DELETE))); + } + + /** Returns the default {@link Environment#dev} {@link ZoneId}, to use for development deployments. */ + public ZoneId devZone() { + Inspector rootObject = toInspector(send(request(HttpRequest.newBuilder(defaultRegionPath()) + .timeout(Duration.ofSeconds(10)), + GET))); + return ZoneId.from("dev", rootObject.field("name").asString()); + } + + /** Returns the Vespa version to compile against, for a hosted Vespa application. This is its lowest runtime version. */ + public String compileVersion(ApplicationId id) { + return toInspector(send(request(HttpRequest.newBuilder(applicationPath(id.tenant(), id.application())) + .timeout(Duration.ofSeconds(10)), + GET))) + .field("compileVersion").asString(); + } + protected HttpRequest request(HttpRequest.Builder request, Method method, Supplier<InputStream> data) { return request.method(method.name(), ofInputStream(data)).build(); } @@ -86,12 +121,12 @@ public abstract class ControllerHttpClient { return request(request.setHeader("Content-Type", data.contentType()), method, data::data); } - private URI apiPath() { + private URI applicationApiPath() { return concatenated(endpoint, "application", "v4"); } private URI tenantPath(TenantName tenant) { - return concatenated(apiPath(), "tenant", tenant.value()); + return concatenated(applicationApiPath(), "tenant", tenant.value()); } private URI applicationPath(TenantName tenant, ApplicationName application) { @@ -102,6 +137,17 @@ public abstract class ControllerHttpClient { return concatenated(applicationPath(id.tenant(), id.application()), "instance", id.instance().value()); } + private URI deploymentPath(ApplicationId id, ZoneId zone) { + return concatenated(applicationPath(id.tenant(), id.application()), + "environment", zone.environment().value(), + "region", zone.region().value(), + "instance", id.instance().value()); + } + + private URI defaultRegionPath() { + return concatenated(endpoint, "zone", "v1", "environment", Environment.dev.value(), "default"); + } + private static URI concatenated(URI base, String... parts) { return base.resolve(String.join("/", parts) + "/"); } @@ -119,34 +165,77 @@ public abstract class ControllerHttpClient { } } + /** Returns a JSON representation of the deployment meta data. */ + private static String metaToJson(Deployment deployment) { + Slime slime = new Slime(); + Cursor rootObject = slime.setObject(); + + if (deployment.repository().isPresent()) { + Cursor revisionObject = rootObject.setObject("sourceRevision"); + deployment.repository().ifPresent(repository -> revisionObject.setString("repository", repository)); + deployment.branch().ifPresent(branch -> revisionObject.setString("branch", branch)); + deployment.commit().ifPresent(commit -> revisionObject.setString("commit", commit)); + deployment.build().ifPresent(build -> rootObject.setLong("buildNumber", build)); + } + + deployment.version().ifPresent(version -> rootObject.setString("vespaVersion", version)); + + if (deployment.ignoreValidationErrors()) rootObject.setBool("ignoreValidationErrors", true); + rootObject.setBool("deployDirectly", true); + + return toJson(slime); + } + /** Returns a JSON representation of the submission meta data. */ private static String metaToJson(Submission submission) { - try { - Slime slime = new Slime(); - Cursor rootObject = slime.setObject(); - rootObject.setString("repository", submission.repository()); - rootObject.setString("branch", submission.branch()); - rootObject.setString("commit", submission.commit()); - rootObject.setString("authorEmail", submission.authorEmail()); - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - new JsonFormat(true).encode(buffer, slime); - return buffer.toString(UTF_8); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } + Slime slime = new Slime(); + Cursor rootObject = slime.setObject(); + rootObject.setString("repository", submission.repository()); + rootObject.setString("branch", submission.branch()); + rootObject.setString("commit", submission.commit()); + rootObject.setString("authorEmail", submission.authorEmail()); + submission.projectId().ifPresent(projectId -> rootObject.setLong("projectId", projectId)); + return toJson(slime); } - /** Returns the "message" element contained in the JSON formatted response, if 2XX status code, or throws otherwise. */ - private static String toMessage(HttpResponse<byte[]> response) { - Inspector rootObject = toSlime(response.body()).get(); - if (response.statusCode() / 100 == 2) - return rootObject.field("message").asString(); + /** Returns a multi part data stream with meta data and, if contained in the deployment, an application package. */ + private static MultiPartStreamer toDataStream(Deployment deployment) { + MultiPartStreamer streamer = new MultiPartStreamer(); + streamer.addJson("deployOptions", metaToJson(deployment)); + deployment.applicationZip().ifPresent(zip -> streamer.addFile("applicationZip", zip)); + return streamer; + } + + private static String asText(HttpResponse<byte[]> response) { + toInspector(response); + return new String(response.body(), UTF_8); + } - else { + /** Returns an {@link Inspector} for the assumed JSON formatted response, or throws if the status code is non-2XX. */ + private static Inspector toInspector(HttpResponse<byte[]> response) { + Inspector rootObject = toSlime(response.body()).get(); + if (response.statusCode() / 100 != 2) throw new RuntimeException(response.request() + " returned code " + response.statusCode() + " (" + rootObject.field("error-code").asString() + "): " + rootObject.field("message").asString()); + + return rootObject; + } + + /** Returns the "message" element contained in the JSON formatted response, if 2XX status code, or throws otherwise. */ + private static String toMessage(HttpResponse<byte[]> response) { + return toInspector(response).field("message").asString(); + } + + private static DeploymentResult toDeploymentResult(HttpResponse<byte[]> response) { + try { + Inspector responseObject = toInspector(response); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + new JsonFormat(false).encode(buffer, responseObject); // Pretty-print until done properly. + return new DeploymentResult(buffer.toString(UTF_8)); + } + catch (IOException e) { + throw new UncheckedIOException(e); } } @@ -154,6 +243,17 @@ public abstract class ControllerHttpClient { return new JsonDecoder().decode(new Slime(), data); } + private static String toJson(Slime slime) { + try { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + new JsonFormat(true).encode(buffer, slime); + return buffer.toString(UTF_8); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + /** Client that signs requests with a private key whose public part is assigned to an application in the remote controller. */ private static class SigningControllerHttpClient extends ControllerHttpClient { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java new file mode 100644 index 00000000000..8f981ca5f05 --- /dev/null +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Deployment.java @@ -0,0 +1,67 @@ +package ai.vespa.hosted.api; + +import java.nio.file.Path; +import java.util.Optional; +import java.util.OptionalLong; + +/** + * A deployment intended for hosted Vespa, containing an application package and some meta data. + */ +public class Deployment { + + // Deployment options + private final Optional<String> version; + private final boolean ignoreValidationErrors; + + // Provide an application package ... + private final Optional<Path> applicationZip; + + // ... or reference a previously submitted one. + private final Optional<String> repository; + private final Optional<String> branch; + private final Optional<String> commit; + private final OptionalLong build; + + private Deployment(Optional<String> version, boolean ignoreValidationErrors, Optional<Path> applicationZip, + Optional<String> repository, Optional<String> branch, Optional<String> commit, OptionalLong build) { + this.version = version; + this.ignoreValidationErrors = ignoreValidationErrors; + this.applicationZip = applicationZip; + this.repository = repository; + this.branch = branch; + this.commit = commit; + this.build = build; + } + + + /** Returns a deployment which will use the provided application package. */ + public static Deployment ofPackage(Path applicationZipFile) { + return new Deployment(Optional.empty(), false, Optional.of(applicationZipFile), + Optional.empty(), Optional.empty(), Optional.empty(), OptionalLong.empty()); + } + + /** Returns a deployment which will use the previously submitted package with the given reference. */ + public static Deployment ofReference(String repository, String branch, String commit, long build) { + return new Deployment(Optional.empty(), false, Optional.empty(), + Optional.of(repository), Optional.of(branch), Optional.of(commit), OptionalLong.of(build)); + } + + /** Returns a copy of this which will have the specified Vespa version on its nodes. */ + public Deployment atVersion(String vespaVersion) { + return new Deployment(Optional.of(vespaVersion), ignoreValidationErrors, applicationZip, repository, branch, commit, build); + } + + /** Returns a copy of this which will additionally ignore validation errors upon deployment. */ + public Deployment ignoringValidationErrors() { + return new Deployment(version, true, applicationZip, repository, branch, commit, build); + } + + public Optional<String> version() { return version; } + public boolean ignoreValidationErrors() { return ignoreValidationErrors; } + public Optional<Path> applicationZip() { return applicationZip; } + public Optional<String> repository() { return repository; } + public Optional<String> branch() { return branch; } + public Optional<String> commit() { return commit; } + public OptionalLong build() { return build; } + +} diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java new file mode 100644 index 00000000000..63142ffbbf1 --- /dev/null +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java @@ -0,0 +1,21 @@ +package ai.vespa.hosted.api; + + +/** + * Contains information about the result of a {@link Deployment} against a {@link ControllerHttpClient}. + * + * @author jonmv + */ +public class DeploymentResult { + + private final String json; // TODO probably do this properly. + + public DeploymentResult(String json) { + this.json = json; + } + + public String json() { + return json; + } + +} diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java index f4cb90176da..8205e8a1b9b 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java @@ -4,9 +4,10 @@ package ai.vespa.hosted.api; import java.io.File; import java.io.InputStream; import java.nio.file.Path; +import java.util.OptionalLong; /** - * A submission intended for hosted Vespa containing an application package with tests and meta data. + * A submission intended for hosted Vespa, containing an application package with tests, and meta data. * * @author jonmv */ @@ -18,14 +19,16 @@ public class Submission { private final String authorEmail; private final Path applicationZip; private final Path applicationTestZip; + private final OptionalLong projectId; - public Submission(String repository, String branch, String commit, String authorEmail, Path applicationZip, Path applicationTestZip) { + public Submission(String repository, String branch, String commit, String authorEmail, Path applicationZip, Path applicationTestZip, OptionalLong projectId) { this.repository = repository; this.branch = branch; this.commit = commit; this.authorEmail = authorEmail; this.applicationZip = applicationZip; this.applicationTestZip = applicationTestZip; + this.projectId = projectId; } public String repository() { return repository; } @@ -34,5 +37,6 @@ public class Submission { public String authorEmail() { return authorEmail; } public Path applicationZip() { return applicationZip; } public Path applicationTestZip() { return applicationTestZip; } + public OptionalLong projectId() { return projectId; } } diff --git a/metrics-proxy/OWNERS b/metrics-proxy/OWNERS deleted file mode 100644 index 3b2ba1ede81..00000000000 --- a/metrics-proxy/OWNERS +++ /dev/null @@ -1 +0,0 @@ -gjoranv diff --git a/metrics-proxy/README b/metrics-proxy/README deleted file mode 100644 index 4008cb1aa5d..00000000000 --- a/metrics-proxy/README +++ /dev/null @@ -1 +0,0 @@ -The metrics proxy provides a single point of access for metrics from all Vespa services. diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml deleted file mode 100644 index 88c250f31bd..00000000000 --- a/metrics-proxy/pom.xml +++ /dev/null @@ -1,157 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>metrics-proxy</artifactId> - <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> - <dependencies> - - <!-- provided --> - - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>annotations</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>component</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-lib</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-di</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jrt</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>yolean</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <scope>provided</scope> - </dependency> - - <!-- compile scope --> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>http-utils</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - - <!-- test scope --> - - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-class-plugin</artifactId> - <version>${project.version}</version> - <executions> - <execution> - <id>config-gen</id> - <goals> - <goal>config-gen</goal> - </goals> - </execution> - </executions> - <configuration> - <packagePrefix>ai.vespa.</packagePrefix> - </configuration> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <Bundle-SymbolicName>${project.groupId}.${project.artifactId}</Bundle-SymbolicName> - </configuration> - </plugin> - - <plugin> - <!-- Only added to make IntelliJ use correct language level --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java deleted file mode 100644 index 564de0806ca..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.core; - -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.metric.model.ConsumerId; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collector; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static com.yahoo.stream.CustomCollectors.toLinkedMap; -import static java.util.Collections.unmodifiableSet; -import static java.util.stream.Collectors.collectingAndThen; - -/** - * Contains metrics consumers and their metrics, and mappings between these. - * All collections are final and immutable. - * - * @author gjoranv - */ -public class MetricsConsumers { - - // All metrics for each consumer. - private final Map<ConsumerId, List<Consumer.Metric>> consumerMetrics; - - // All consumers for each metric (more useful than the opposite map). - private final Map<Consumer.Metric, List<ConsumerId>> consumersByMetric; - - public MetricsConsumers(ConsumersConfig config) { - consumerMetrics = config.consumer().stream().collect( - toUnmodifiableLinkedMap(consumer -> toConsumerId(consumer.name()), Consumer::metric)); - - consumersByMetric = createConsumersByMetric(consumerMetrics); - } - - /** - * @param consumer The consumer - * @return The metrics for the given consumer. - */ - public List<Consumer.Metric> getMetricDefinitions(ConsumerId consumer) { - return consumerMetrics.get(consumer); - } - - public Map<Consumer.Metric, List<ConsumerId>> getConsumersByMetric() { - return consumersByMetric; - } - - public Set<ConsumerId> getAllConsumers() { - return unmodifiableSet(consumerMetrics.keySet()); - } - - /** - * Helper function to create mapping from metric to consumers. - * TODO: consider reversing the mapping in metrics-consumers.def instead: metric{}.consumer[] - */ - private static Map<Consumer.Metric, List<ConsumerId>> - createConsumersByMetric(Map<ConsumerId, List<Consumer.Metric>> metricsByConsumer) { - Map<Consumer.Metric, List<ConsumerId>> consumersByMetric = new LinkedHashMap<>(); - metricsByConsumer.forEach( - (consumer, metrics) -> metrics.forEach( - metric -> consumersByMetric.computeIfAbsent(metric, unused -> new ArrayList<>()) - .add(consumer))); - return Collections.unmodifiableMap(consumersByMetric); - } - - public static <T, K, U> Collector<T, ?, Map<K, U>> toUnmodifiableLinkedMap(Function<? super T, ? extends K> keyMapper, - Function<? super T, ? extends U> valueMapper) { - return collectingAndThen(toLinkedMap(keyMapper, valueMapper), Collections::unmodifiableMap); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java deleted file mode 100644 index fe823c72127..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.core; - -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; -import com.yahoo.component.Vtag; - -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static ai.vespa.metricsproxy.metric.ExternalMetrics.extractConfigserverDimensions; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static com.yahoo.log.LogLevel.DEBUG; -import static java.util.stream.Collectors.toList; - -/** - * Retrieves metrics and performs necessary conversions and additions of metadata. - * - * @author gjoranv - */ -public class MetricsManager { - private static Logger log = Logger.getLogger(MetricsManager.class.getName()); - - static final DimensionId VESPA_VERSION = toDimensionId("vespaVersion"); - - private final VespaServices vespaServices; - private final VespaMetrics vespaMetrics; - private final ExternalMetrics externalMetrics; - private final ApplicationDimensions applicationDimensions; - private final NodeDimensions nodeDimensions; - - private volatile Map<DimensionId, String> extraDimensions = new HashMap<>(); - - public MetricsManager(VespaServices vespaServices, - VespaMetrics vespaMetrics, - ExternalMetrics externalMetrics, - ApplicationDimensions applicationDimensions, - NodeDimensions nodeDimensions) { - this.vespaServices = vespaServices; - this.vespaMetrics = vespaMetrics; - this.externalMetrics = externalMetrics; - this.applicationDimensions = applicationDimensions; - this.nodeDimensions = nodeDimensions; - } - - /** - * Returns all metrics for the given service that are whitelisted for the given consumer. - */ - public String getMetricNamesForServiceAndConsumer(String service, ConsumerId consumer) { - return vespaMetrics.getMetricNames(vespaServices.getMonitoringServices(service), consumer); - } - - public String getMetricsByConfigId(String configId) { - List<VespaService> services = vespaServices.getInstancesById(configId); - vespaServices.updateServices(services); - - return vespaMetrics.getMetricsAsString(services); - } - - /** - * Returns the metrics for the given services. The empty list is returned if no services are given. - * - * @param services The services to retrieve metrics for. - * @return Metrics for all matching services. - */ - public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) { - if (services.isEmpty()) return Collections.emptyList(); - vespaServices.updateServices(services); - - List<MetricsPacket.Builder> result = vespaMetrics.getMetrics(services); - log.log(DEBUG, () -> "Got " + result.size() + " metrics packets for vespa services."); - - List<MetricsPacket.Builder> externalPackets = externalMetrics.getMetrics().stream() - .filter(MetricsPacket.Builder::hasMetrics) - .collect(toList()); - log.log(DEBUG, () -> "Got " + externalPackets.size() + " external metrics packets with whitelisted metrics."); - - result.addAll(externalPackets); - - return result.stream() - .map(builder -> builder.putDimensionsIfAbsent(getGlobalDimensions())) - .map(builder -> builder.putDimensionsIfAbsent(extraDimensions)) - .map(builder -> adjustTimestamp(builder, startTime)) - .map(MetricsPacket.Builder::build) - .collect(Collectors.toList()); - } - - /** - * Returns a merged map of all global dimensions. - */ - private Map<DimensionId, String> getGlobalDimensions() { - Map<DimensionId, String> globalDimensions = new LinkedHashMap<>(applicationDimensions.getDimensions()); - globalDimensions.putAll(nodeDimensions.getDimensions()); - globalDimensions.put(VESPA_VERSION, Vtag.currentVersion.toFullString()); - return globalDimensions; - } - - /** - * If the metrics packet is less than one minute newer or older than the given startTime, - * set its timestamp to the given startTime. This is done to ensure that metrics retrieved - * from different sources for this invocation get the same timestamp, and a timestamp as close - * as possible to the invocation from the external metrics retrieving client. The assumption - * is that the client requests metrics periodically every minute. - * <p> - * However, if the timestamp of the packet is too far off in time, we don't adjust it because - * we would otherwise be masking a real problem with retrieving the metrics. - */ - static MetricsPacket.Builder adjustTimestamp(MetricsPacket.Builder builder, Instant startTime) { - Duration age = Duration.between(startTime, builder.getTimestamp()); - if (age.abs().minusMinutes(1).isNegative()) - builder.timestamp(startTime.getEpochSecond()); - return builder; - } - - /** - * Returns the health metrics for the given services. The empty list is returned if no services are given. - * - * @param services The services to retrieve health metrics for. - * @return Health metrics for all matching services. - */ - public List<MetricsPacket> getHealthMetrics(List<VespaService> services) { - if (services.isEmpty()) return Collections.emptyList(); - vespaServices.updateServices(services); - - // TODO: Add global dimensions to health metrics? - return vespaMetrics.getHealthMetrics(services); - } - - public void setExtraMetrics(List<MetricsPacket.Builder> packets) { - extraDimensions = extractConfigserverDimensions(packets); - externalMetrics.setExtraMetrics(packets); - } - - /** - * Returns a space separated list of all distinct service names. - */ - public String getAllVespaServices() { - return vespaServices.getVespaServices().stream() - .map(VespaService::getServiceName) - .distinct() - .collect(Collectors.joining(" ")); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java deleted file mode 100644 index becfd9a54ce..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ /dev/null @@ -1,307 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.core; - - -import ai.vespa.metricsproxy.metric.AggregationKey; -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.MetricsFormatter; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static com.google.common.base.Strings.isNullOrEmpty; -import static com.yahoo.log.LogLevel.DEBUG; - -/** - * @author Unknown - * @author gjoranv - */ -public class VespaMetrics { - private static final Logger log = Logger.getLogger(VespaMetrics.class.getPackage().getName()); - - // MUST be the same as the constant defined in config-model - public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa"); - - public static final DimensionId METRIC_TYPE_DIMENSION_ID = toDimensionId("metrictype"); - public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId("instance"); - - private static final Set<ConsumerId> DEFAULT_CONSUMERS = Collections.singleton(VESPA_CONSUMER_ID); - - private final MetricsConsumers metricsConsumers; - - private static final MetricsFormatter formatter = new MetricsFormatter(false, false); - - public VespaMetrics(MetricsConsumers metricsConsumers, VespaServices vespaServices) { - this.metricsConsumers = metricsConsumers; - } - - public List<MetricsPacket> getHealthMetrics(List<VespaService> services) { - List<MetricsPacket> result = new ArrayList<>(); - for (VespaService s : services) { - HealthMetric h = s.getHealth(); - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(s.getMonitoringName())) - .statusCode(h.isOk() ? 0 : 1) - .statusMessage(h.getMessage()) - .putDimension(METRIC_TYPE_DIMENSION_ID, "health") - .putDimension(INSTANCE_DIMENSION_ID, s.getInstanceName()); - - result.add(builder.build()); - } - - return result; - } - - /** - * @param services The services to get metrics for - * @return A list of metrics packet builders (to allow modification by the caller). - */ - public List<MetricsPacket.Builder> getMetrics(List<VespaService> services) { - List<MetricsPacket.Builder> metricsPackets = new ArrayList<>(); - - log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); - - Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric = metricsConsumers.getConsumersByMetric(); - - for (VespaService service : services) { - // One metrics packet for system metrics - Optional<MetricsPacket.Builder> systemCheck = getSystemMetrics(service); - systemCheck.ifPresent(metricsPackets::add); - - // One metrics packet per set of metrics that share the same dimensions+consumers - // TODO: Move aggregation into MetricsPacket itself? - Metrics serviceMetrics = getServiceMetrics(service, consumersByMetric); - Map<AggregationKey, List<Metric>> aggregatedMetrics = - aggregateMetrics(service.getDimensions(), serviceMetrics); - - aggregatedMetrics.forEach((aggregationKey, metrics) -> { - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(service.getMonitoringName())) - .putMetrics(metrics) - .putDimension(METRIC_TYPE_DIMENSION_ID, "standard") - .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName()) - .putDimensions(aggregationKey.getDimensions()); - setMetaInfo(builder, serviceMetrics.getTimeStamp()); - builder.addConsumers(aggregationKey.getConsumers()); - metricsPackets.add(builder); - }); - } - - return metricsPackets; - } - - /** - * Returns the metrics to output for the given service, with updated timestamp - * In order to include a metric, it must exist in the given map of metric to consumers. - * Each returned metric will contain a collection of consumers that it should be routed to. - */ - private Metrics getServiceMetrics(VespaService service, Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric) { - Metrics serviceMetrics = new Metrics(); - Metrics allServiceMetrics = service.getMetrics(); - serviceMetrics.setTimeStamp(getMostRecentTimestamp(allServiceMetrics)); - for (Metric candidate : allServiceMetrics.getMetrics()) { - getConfiguredMetrics(candidate.getName(), consumersByMetric.keySet()).forEach( - configuredMetric -> serviceMetrics.add( - metricWithConfigProperties(candidate, configuredMetric, consumersByMetric))); - } - return serviceMetrics; - } - - private Map<DimensionId, String> extractDimensions(Map<DimensionId, String> dimensions, List<ConsumersConfig.Consumer.Metric.Dimension> configuredDimensions) { - if ( ! configuredDimensions.isEmpty()) { - Map<DimensionId, String> dims = new HashMap<>(dimensions); - configuredDimensions.forEach(d -> dims.put(toDimensionId(d.key()), d.value())); - dimensions = Collections.unmodifiableMap(dims); - } - return dimensions; - } - - private Set<ConsumerId> extractConsumers(List<ConsumerId> configuredConsumers) { - Set<ConsumerId> consumers = Collections.emptySet(); - if (configuredConsumers != null) { - if ( configuredConsumers.size() == 1) { - consumers = Collections.singleton(configuredConsumers.get(0)); - } else if (configuredConsumers.size() > 1){ - consumers = new HashSet<>(); - consumers.addAll(configuredConsumers); - consumers = Collections.unmodifiableSet(consumers); - } - } - return consumers; - } - - private Metric metricWithConfigProperties(Metric candidate, - ConsumersConfig.Consumer.Metric configuredMetric, - Map<ConsumersConfig.Consumer.Metric, List<ConsumerId>> consumersByMetric) { - Metric metric = candidate.clone(); - metric.setDimensions(extractDimensions(candidate.getDimensions(), configuredMetric.dimension())); - metric.setConsumers(extractConsumers(consumersByMetric.get(configuredMetric))); - - if (!isNullOrEmpty(configuredMetric.outputname())) - metric.setName(configuredMetric.outputname()); - return metric; - } - - /** - * Returns all configured metrics (for any consumer) that have the given id as 'name'. - */ - private static Set<ConsumersConfig.Consumer.Metric> getConfiguredMetrics(String id, - Set<ConsumersConfig.Consumer.Metric> configuredMetrics) { - return configuredMetrics.stream() - .filter(m -> m.name().equals(id)) - .collect(Collectors.toSet()); - } - - private Optional<MetricsPacket.Builder> getSystemMetrics(VespaService service) { - Metrics systemMetrics = service.getSystemMetrics(); - if (systemMetrics.size() == 0) return Optional.empty(); - - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(service.getMonitoringName())); - setMetaInfo(builder, systemMetrics.getTimeStamp()); - - builder.putDimension(METRIC_TYPE_DIMENSION_ID, "system") - .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName()) - .putDimensions(service.getDimensions()) - .putMetrics(systemMetrics.getMetrics()); - - builder.addConsumers(metricsConsumers.getAllConsumers()); - return Optional.of(builder); - } - - private long getMostRecentTimestamp(Metrics metrics) { - long mostRecentTimestamp = 0L; - for (Metric metric : metrics.getMetrics()) { - if (metric.getTimeStamp() > mostRecentTimestamp) { - mostRecentTimestamp = metric.getTimeStamp(); - } - } - return mostRecentTimestamp; - } - - private Map<AggregationKey, List<Metric>> aggregateMetrics(Map<DimensionId, String> serviceDimensions, - Metrics metrics) { - Map<AggregationKey, List<Metric>> aggregatedMetrics = new HashMap<>(); - - for (Metric metric : metrics.getMetrics() ) { - Map<DimensionId, String> mergedDimensions = new LinkedHashMap<>(); - mergedDimensions.putAll(metric.getDimensions()); - mergedDimensions.putAll(serviceDimensions); - AggregationKey aggregationKey = new AggregationKey(mergedDimensions, metric.getConsumers()); - - if (aggregatedMetrics.containsKey(aggregationKey)) { - aggregatedMetrics.get(aggregationKey).add(metric); - } else { - List<Metric> ml = new ArrayList<>(); - ml.add(metric); - aggregatedMetrics.put(aggregationKey, ml); - } - } - return aggregatedMetrics; - } - - private List<ConsumersConfig.Consumer.Metric> getMetricDefinitions(ConsumerId consumer) { - if (metricsConsumers == null) return Collections.emptyList(); - - List<ConsumersConfig.Consumer.Metric> definitions = metricsConsumers.getMetricDefinitions(consumer); - return definitions == null ? Collections.emptyList() : definitions; - } - - private static void setMetaInfo(MetricsPacket.Builder builder, long timestamp) { - builder.timestamp(timestamp) - .statusCode(0) - .statusMessage("Data collected successfully"); - } - - /** - * Returns a string representation of metrics for the given services; - * a space separated list of key=value. - */ - public String getMetricsAsString(List<VespaService> services) { - StringBuilder b = new StringBuilder(); - for (VespaService s : services) { - for (Metric metric : s.getMetrics().getMetrics()) { - String key = metric.getName(); - String alias = key; - - boolean isForwarded = false; - for (ConsumersConfig.Consumer.Metric metricConsumer : getMetricDefinitions(VESPA_CONSUMER_ID)) { - if (metricConsumer.name().equals(key)) { - alias = metricConsumer.outputname(); - isForwarded = true; - } - } - if (isForwarded) { - b.append(formatter.format(s, alias, metric.getValue())) - .append(" "); - } - } - } - return b.toString(); - } - - /** - * Get all metric names for the given services - * - * @return String representation - */ - public String getMetricNames(List<VespaService> services, ConsumerId consumer) { - StringBuilder bufferOn = new StringBuilder(); - StringBuilder bufferOff = new StringBuilder(); - for (VespaService s : services) { - - for (Metric m : s.getMetrics().getMetrics()) { - String description = m.getDescription(); - String alias = ""; - boolean isForwarded = false; - - for (ConsumersConfig.Consumer.Metric metric : getMetricDefinitions(consumer)) { - if (metric.name().equals(m.getName())) { - alias = metric.outputname(); - isForwarded = true; - if (description.isEmpty()) { - description = metric.description(); - } - } - } - - String message = "OFF"; - StringBuilder buffer = bufferOff; - if (isForwarded) { - buffer = bufferOn; - message = "ON"; - } - buffer.append(m.getName()).append('=').append(message); - if (!description.isEmpty()) { - buffer.append(";description=").append(description); - } - if (!alias.isEmpty()) { - buffer.append(";output-name=").append(alias); - } - buffer.append(','); - } - } - - return bufferOn.toString() + bufferOff.toString(); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java deleted file mode 100644 index 617cf0a1525..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -@ExportPackage -package ai.vespa.metricsproxy.core; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java deleted file mode 100644 index 9eb1b242535..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -/** - * @author gjoranv - */ -public final class AggregationKey { - - private Map<DimensionId, String> dimensions; - private Set<ConsumerId> consumers; - - public AggregationKey(Map<DimensionId, String> dimensions, Set<ConsumerId> consumers) { - this.dimensions = dimensions; - this.consumers = consumers; - } - - public Map<DimensionId, String> getDimensions() { return Collections.unmodifiableMap(dimensions); } - - public Set<ConsumerId> getConsumers() { return Collections.unmodifiableSet(consumers); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AggregationKey that = (AggregationKey) o; - return Objects.equals(dimensions, that.dimensions) && - Objects.equals(consumers, that.consumers); - } - - @Override - public int hashCode() { - return Objects.hash(dimensions, consumers); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java deleted file mode 100644 index 62465909798..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.ServiceId; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; - -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static com.yahoo.log.LogLevel.DEBUG; -import static java.util.stream.Collectors.toCollection; - -/** - * This class is responsible for handling metrics received from external processes. - * - * @author gjoranv - */ -public class ExternalMetrics { - private static final Logger log = Logger.getLogger(ExternalMetrics.class.getName()); - - public static final DimensionId ROLE_DIMENSION = toDimensionId("role"); - public static final DimensionId STATE_DIMENSION = toDimensionId("state"); - public static final DimensionId ORCHESTRATOR_STATE_DIMENSION = toDimensionId("orchestratorState"); - - static final ServiceId VESPA_NODE_SERVICE_ID = toServiceId("vespa.node"); - - private volatile List<MetricsPacket.Builder> metrics = new ArrayList<>(); - private final MetricsConsumers consumers; - - public ExternalMetrics(MetricsConsumers consumers) { - this.consumers = consumers; - } - - public List<MetricsPacket.Builder> getMetrics() { - return metrics; - } - - public void setExtraMetrics(List<MetricsPacket.Builder> externalPackets) { - log.log(DEBUG, () -> "Setting new external metrics with " + externalPackets.size() + " metrics packets."); - externalPackets.forEach(packet -> { - packet.addConsumers(consumers.getAllConsumers()) - .service(VESPA_NODE_SERVICE_ID) - .retainMetrics(metricsToRetain()) - .applyOutputNames(outputNamesById()); - }); - metrics = List.copyOf(externalPackets); - } - - private Set<MetricId> metricsToRetain() { - return consumers.getConsumersByMetric().keySet().stream() - .map(configuredMetric -> toMetricId(configuredMetric.name())) - .collect(toCollection(LinkedHashSet::new)); - } - - /** - * Returns a mapping from metric id to a list of the metric's output names. - * Metrics that only have their id as output name are included in the output. - */ - private Map<MetricId, List<String>> outputNamesById() { - Map<MetricId, List<String>> outputNamesById = new LinkedHashMap<>(); - for (Consumer.Metric metric : consumers.getConsumersByMetric().keySet()) { - MetricId id = toMetricId(metric.name()); - outputNamesById.computeIfAbsent(id, unused -> new ArrayList<>()) - .add(metric.outputname()); - } - return outputNamesById; - } - - /** - * Extracts the node repository dimensions (role, state etc.) from the given packets. - * If the same dimension exists in multiple packets, this implementation gives no guarantees - * about which value is returned. - */ - public static Map<DimensionId, String> extractConfigserverDimensions(Collection<MetricsPacket.Builder> packets) { - Map<DimensionId, String> dimensions = new HashMap<>(); - for (MetricsPacket.Builder packet : packets) { - dimensions.putAll(packet.build().dimensions()); - } - dimensions.keySet().retainAll(Set.of(ROLE_DIMENSION, STATE_DIMENSION, ORCHESTRATOR_STATE_DIMENSION)); - return dimensions; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java deleted file mode 100644 index 41a8c3d414e..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java +++ /dev/null @@ -1,56 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -/** - * @author Jo Kristian Bergum - */ -public class HealthMetric { - private final String message; - private final String status; - private final boolean isAlive; - - private HealthMetric(String status, String message, boolean isAlive) { - this.message = message; - this.status = status; - this.isAlive = isAlive; - } - - public static HealthMetric get(String status, String message) { - if (status == null) { - status = ""; - } - if (message == null) { - message = ""; - } - status = status.toLowerCase(); - - if (status.equals("up") || status.equals("ok")) { - return new HealthMetric(status, message, true); - } else { - return new HealthMetric(status, message, false); - } - } - - public static HealthMetric getFailed(String message) { - return new HealthMetric("down", message, false); - } - - public static HealthMetric getOk(String message) { - return new HealthMetric("up", message, true); - } - - public String getMessage() { - return this.message; - } - - public String getStatus() { - return this.status; - } - - public boolean isOk() { - return this.isAlive; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java deleted file mode 100644 index 59fbe301a49..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java +++ /dev/null @@ -1,124 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - -/** - * @author Jo Kristian Bergum - */ -public class Metric { - private final long time; - private final Number value; - private final String description; - private String name; - private Map<DimensionId, String> dimensions; - private Set<ConsumerId> consumers; - - /** - * Creates a new metric instance - * - * @param name The metric name. E.g 'documents' - * @param value The numeric value - * @param time The timestamp of this metric in seconds - */ - public Metric(String name, Number value, long time, Map<DimensionId, String> dimensions, String description) { - this.time = time; - this.value = value; - this.name = name; - this.dimensions = dimensions; - this.description = description; - } - - public Metric(String name, Number value, long timestamp) { - this(name, value, timestamp, Collections.emptyMap(), ""); - } - - public Metric(String name, Number value) { - this(name, value, System.currentTimeMillis() / 1000); - } - - public void setDimensions(Map<DimensionId, String> dimensions) { - this.dimensions = dimensions; - } - - /** - * @return A map of the dimensions registered for this metric - */ - public Map<DimensionId, String> getDimensions() { return dimensions; } - - public void setConsumers(Set<ConsumerId> consumers) { this.consumers = consumers; } - - /** - * @return The consumers this metric should be routed to. - */ - public Set<ConsumerId> getConsumers() { return consumers; } - - /** - * @return The number that this metric name represent - */ - public Number getValue() { - return value; - } - - /** - * Set the name of this metric - * - * @param name The name to use for this metric - */ - public void setName(String name) { - this.name = name; - } - - /** - * @return The name of the metric - */ - public String getName() { - return name; - } - - /** - * @return The UTC timestamp for when this metric was collected - */ - public long getTimeStamp() { - return this.time; - } - - @Override - public String toString() { - return "Metric{" + - "time=" + time + - ", name=" + name + - ", value='" + value + '\'' + - ", dimensions=" + dimensions + - '}'; - } - - @Override - public Metric clone() { - return new Metric(name, value, time, new LinkedHashMap<>(dimensions), getDescription()); - } - - /** - * @return the description of this metric - */ - public String getDescription() { - return this.description; - } - - /** Return an adjusted (rounded up) time if necessary */ - public static long adjustTime(long timestamp, long now) { - if ((now == (timestamp+1)) && ((now % 60) == 0)) { - return now; - } - return timestamp; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java deleted file mode 100644 index ca611368730..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java +++ /dev/null @@ -1,111 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Once a getter is called, the instance is frozen and no more metrics can be added. - * - * @author Unknown - */ -// TODO: remove timestamp, only used as temporary storage. -// TODO: instances of this class can probably be replaced by a simple freezable map. -public class Metrics { - private final List<Metric> metrics = new ArrayList<>(); - private long timestamp; - private boolean isFrozen = false; - - public Metrics() { - this(System.currentTimeMillis() / 1000L); - } - - public Metrics(long timestamp) { - this.timestamp = timestamp; - } - - private void ensureNotFrozen() { - if (isFrozen) throw new IllegalStateException("Frozen Metrics cannot be modified!"); - - } - - public long getTimeStamp() { - return this.timestamp; - } - - /** - * Update the timestamp - * - * @param timestamp IN UTC seconds resolution - */ - public void setTimeStamp(long timestamp) { - ensureNotFrozen(); - this.timestamp = timestamp; - } - - public void add(Metric m) { - ensureNotFrozen(); - this.timestamp = m.getTimeStamp(); - this.metrics.add(m); - } - - /** - * Get the size of the metrics covered. Note that this might also contain expired metrics - * - * @return size of metrics - */ - public int size() { - return this.metrics.size(); - } - - /** - * TODO: Remove, might be multiple metrics with same name but different dimensions - * - * @param key metric name - * @return the metric, or null - */ - public Metric getMetric(String key) { - isFrozen = true; - for (Metric m: metrics) { - if (m.getName().equals(key)) { - return m; - } - } - return null; - } - - public List<Metric> getMetrics() { - isFrozen = true; - return Collections.unmodifiableList(metrics); - } - - - /** - * Get a single metric based on the metric name - * TODO: Remove, might be multiple metrics with same name, but different - * - * @param key metric name - * @return The value or null if metric was not found or expired - */ - public Number get(String key) { - isFrozen = true; - Metric m = getMetric(key); - if (m != null) { - return m.getValue(); - } - return null; - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - for (Metric m : metrics) { - sb.append(m.getName()).append(":").append(m.getValue()).append("\n"); - } - return sb.toString(); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java deleted file mode 100644 index 8858e21486a..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.service.VespaService; - -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.Locale; - -/** - * Format metrics as required by users of the "getMetricsById" rpc method. - * - * @author Unknown - */ -public class MetricsFormatter { - private final boolean includeServiceName; - private final boolean isSystemMetric; - private final DecimalFormat df = new DecimalFormat("0.000", new DecimalFormatSymbols(Locale.ENGLISH)); - - public MetricsFormatter(boolean includeServiceName, boolean isSystemMetric) { - this.includeServiceName = includeServiceName; - this.isSystemMetric = isSystemMetric; - } - - public String format(VespaService service, String name, Number value) { - StringBuilder sb = new StringBuilder(); - - if (includeServiceName) { - sb.append(service.getServiceName()).append("."); - } - - if (isSystemMetric) - sb.append(toSystemServiceId(service.getConfigId())); - else - sb.append(toServiceId(service.getConfigId())); - - sb.append(".") - .append(formatMetricName(name)) - .append("="); - - if (value instanceof Double) { - sb.append(df.format(value.doubleValue())); - } else { - sb.append(value.toString()); - } - - return sb.toString(); - } - - private static String formatMetricName(String name) { - name = name.replaceAll("\"", ""); - name = name.replaceAll("\\.", "_"); - return name; - } - - // E.g. container/qrserver.1 -> 'container.qrserver.1' - private static String toServiceId(String configId) { - return "'" + configId.replace("/", ".") + "'"; - } - - // E.g. container/qrserver.1 -> container.'qrserver.1' - private static String toSystemServiceId(String configId) { - String name = configId.replace("/", "."); - name = name.replaceFirst("\\.", ".'") + "'"; - return name; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java deleted file mode 100644 index ae40f672a32..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.dimensions; - -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Map; - -import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; - -/** - * Application-specific but node-agnostic dimensions. - * - * @author gjoranv - */ -public class ApplicationDimensions { - - private final Map<DimensionId, String> dimensions; - - public ApplicationDimensions(ApplicationDimensionsConfig config) { - dimensions = config.dimensions().entrySet().stream().collect( - toUnmodifiableLinkedMap(e -> toDimensionId(e.getKey()), Map.Entry::getValue)); - } - - public Map<DimensionId, String> getDimensions() { return dimensions; } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java deleted file mode 100644 index d2c1799e148..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.dimensions; - -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Map; - -import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; - -/** - * Node-specific metric dimensions. - * - * @author gjoranv - */ -public class NodeDimensions { - - private final Map<DimensionId, String> dimensions; - - public NodeDimensions(NodeDimensionsConfig config) { - dimensions = config.dimensions().entrySet().stream().collect( - toUnmodifiableLinkedMap(e -> toDimensionId(e.getKey()), Map.Entry::getValue)); - } - - public Map<DimensionId, String> getDimensions() { return dimensions; } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java deleted file mode 100644 index f4e5f74313a..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -@ExportPackage -package ai.vespa.metricsproxy.metric.dimensions; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java deleted file mode 100644 index 0d7acd5f354..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java +++ /dev/null @@ -1,38 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import java.util.Objects; - -/** - * @author gjoranv - */ -public class ConsumerId { - public final String id; - private ConsumerId(String id) { this.id = id; } - - public static ConsumerId toConsumerId(String id) { return new ConsumerId(id); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ConsumerId that = (ConsumerId) o; - return Objects.equals(id, that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "ConsumerId{" + - "id='" + id + '\'' + - '}'; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java deleted file mode 100644 index 03f4c2c01ff..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java +++ /dev/null @@ -1,38 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import java.util.Objects; - -/** - * @author gjoranv - */ -public class DimensionId { - - public final String id; - private DimensionId(String id) { this.id = id; } - - public static DimensionId toDimensionId(String id) { return new DimensionId(id); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DimensionId that = (DimensionId) o; - return Objects.equals(id, that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "DimensionId{" + - "id='" + id + '\'' + - '}'; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java deleted file mode 100644 index c93735c7fca..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java +++ /dev/null @@ -1,39 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import java.util.Objects; - -/** - * @author gjoranv - */ -public class MetricId { - - public final String id; - private MetricId(String id) { this.id = id; } - - public static MetricId toMetricId(String id) { return new MetricId(id); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - MetricId metricId = (MetricId) o; - return Objects.equals(id, metricId.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "MetricId{" + - "id='" + id + '\'' + - '}'; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java deleted file mode 100644 index fa45c6251f6..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import ai.vespa.metricsproxy.metric.Metric; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; - -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static java.util.Collections.unmodifiableList; -import static java.util.Collections.unmodifiableMap; -import static java.util.stream.Collectors.joining; - -/** - * Represents a packet of metrics (with meta information) that belong together because they: - * <ul> - * <li>share both the same dimensions and consumers, AND</li> - * <li>represent the same source, e.g. a vespa service or the system hardware.</li> - * </ul> - * - * @author gjoranv - */ -public class MetricsPacket { - public final int statusCode; - public final String statusMessage; - public final long timestamp; - public final ServiceId service; - private final Map<MetricId, Number> metrics; - private final Map<DimensionId, String> dimensions; - private final List<ConsumerId> consumers; - - private MetricsPacket(int statusCode, String statusMessage, long timestamp, ServiceId service, - Map<MetricId, Number> metrics, Map<DimensionId, String> dimensions, Set<ConsumerId> consumers ) { - this.statusCode = statusCode; - this.statusMessage = statusMessage; - this.timestamp = timestamp; - this.service = service; - this.metrics = metrics; - this.dimensions = dimensions; - this.consumers = new ArrayList<>(consumers); - } - - public Map<MetricId, Number> metrics() { - return unmodifiableMap(metrics); - } - - public Map<DimensionId, String> dimensions() { - return unmodifiableMap(dimensions); - } - - public List<ConsumerId> consumers() { - return unmodifiableList(consumers); - } - - @Override - public String toString() { - return "MetricsPacket{" + - "statusCode=" + statusCode + - ", statusMessage='" + statusMessage + '\'' + - ", timestamp=" + timestamp + - ", service=" + service.id + - ", metrics=" + idMapToString(metrics, id -> id.id) + - ", dimensions=" + idMapToString(dimensions, id -> id.id) + - ", consumers=" + consumers.stream().map(id -> id.id).collect(joining(",", "[", "]")) + - '}'; - } - - private static <K,V> String idMapToString(Map<K,V> map, Function<K, String> idMapper) { - return map.entrySet().stream() - .map(entry -> idMapper.apply(entry.getKey()) + "=" + entry.getValue()) - .collect(joining(",", "{", "}")); - } - - public static class Builder { - // Set sensible defaults here, and use null guard in all setters. - // Except for 'service' for which we require an explicit non-null value. - private ServiceId service; - private int statusCode = 0; - private String statusMessage = "<null>"; - private long timestamp = 0L; - private Map<MetricId, Number> metrics = new LinkedHashMap<>(); - private final Map<DimensionId, String> dimensions = new LinkedHashMap<>(); - private final Set<ConsumerId> consumers = new LinkedHashSet<>(); - - public Builder(ServiceId service) { - Objects.requireNonNull(service, "Service cannot be null."); - this.service = service; - } - - public Builder service(ServiceId service) { - if (service == null) throw new IllegalArgumentException("Service cannot be null."); - this.service = service; - return this; - } - - public Builder statusCode(Integer statusCode) { - if (statusCode != null) this.statusCode = statusCode; - return this; - } - - public Builder statusMessage(String statusMessage) { - if (statusMessage != null) this.statusMessage = statusMessage; - return this; - } - - public Builder timestamp(Long timestamp) { - if (timestamp != null) this.timestamp = timestamp; - return this; - } - - public Builder putMetrics(Collection<Metric> extraMetrics) { - if (extraMetrics != null) - extraMetrics.forEach(metric -> metrics.put(toMetricId(metric.getName()), - metric.getValue().doubleValue())); - return this; - } - - public Builder putMetric(MetricId id, Number value) { - metrics.put(id, value); - return this; - } - - public Builder retainMetrics(Set<MetricId> idsToRetain) { - metrics.keySet().retainAll(idsToRetain); - return this; - } - - public Builder applyOutputNames(Map<MetricId, List<String>> outputNamesById) { - Map<MetricId, Number> newMetrics = new LinkedHashMap<>(); - outputNamesById.forEach((id, outputNames) -> { - if (metrics.containsKey(id)) - outputNames.forEach(outputName -> newMetrics.put(toMetricId(outputName), metrics.get(id))); - }); - metrics = newMetrics; - return this; - } - - public Builder putDimension(DimensionId id, String value) { - dimensions.put(id, value); - return this; - } - - public Builder putDimensions(Map<DimensionId, String> extraDimensions) { - if (extraDimensions != null) dimensions.putAll(extraDimensions); - return this; - } - - public Builder putDimensionsIfAbsent(Map<DimensionId, String> extraDimensions) { - if (extraDimensions != null) extraDimensions.forEach(dimensions::putIfAbsent); - return this; - } - - public Builder addConsumers(Set<ConsumerId> extraConsumers) { - if (extraConsumers != null) consumers.addAll(extraConsumers); - return this; - } - - public MetricsPacket build() { - return new MetricsPacket(statusCode, statusMessage, timestamp, service, metrics, dimensions, consumers); - } - - public boolean hasMetrics() { - return ! metrics.isEmpty(); - } - - public Instant getTimestamp() { return Instant.ofEpochSecond(timestamp); } - - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java deleted file mode 100644 index b61ead75b72..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java +++ /dev/null @@ -1,39 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import java.util.Objects; - -/** - * @author gjoranv - */ -public class ServiceId { - - public final String id; - private ServiceId(String id) { this.id = id; } - - public static ServiceId toServiceId(String id) { return new ServiceId(id); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceId serviceId = (ServiceId) o; - return Objects.equals(id, serviceId.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "ServiceId{" + - "id='" + id + '\'' + - '}'; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java deleted file mode 100644 index f48e5759528..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableMap; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static com.yahoo.stream.CustomCollectors.toLinkedMap; -import static java.util.Collections.emptyList; -import static java.util.logging.Level.WARNING; - -/** - * @author gjoranv - */ -public class JsonUtil { - private static final Logger log = Logger.getLogger(JsonUtil.class.getName()); - - static final String YAMAS_ROUTING = "yamas"; - - public static MetricsPacket.Builder toMetricsPacketBuilder(YamasJsonModel jsonModel) { - if (jsonModel.application == null) - throw new IllegalArgumentException("Service id cannot be null"); - - return new MetricsPacket.Builder(toServiceId(jsonModel.application)) - .statusCode(jsonModel.status_code) - .statusMessage(jsonModel.status_msg) - .timestamp(jsonModel.timestamp) - .putMetrics(jsonModel.getMetricsList()) - .putDimensions(jsonModel.getDimensionsById()) - .addConsumers(jsonModel.getYamasConsumers()); - } - - public static YamasArrayJsonModel toYamasArray(Collection<MetricsPacket> metricsPackets) { - YamasArrayJsonModel yamasArray = toYamasArray(metricsPackets, false); - - // Add a single status object at the end - yamasArray.metrics.stream().findFirst().map(YamasJsonModel::getYamasConsumers) - .ifPresent(consumers -> yamasArray.add(getStatusYamasModel("Data collected successfully", 0, consumers))); - return yamasArray; - } - - public static YamasArrayJsonModel toYamasArray(Collection<MetricsPacket> metricsPackets, boolean addStatus) { - YamasArrayJsonModel yamasArray = new YamasArrayJsonModel(); - metricsPackets.forEach(packet -> yamasArray.add(toYamasModel(packet, addStatus))); - return yamasArray; - } - - /** - * Converts the given json formatted string to a list of metrics packet builders. - * Note that this method returns an empty list if an IOException occurs, - * and logs a warning as a side effect. - */ - public static List<MetricsPacket.Builder> toMetricsPackets(String jsonString) { - List<MetricsPacket.Builder> packets = new ArrayList<>(); - try { - JsonParser jp = new JsonFactory().createParser(jsonString); - jp.setCodec(new ObjectMapper()); - while (jp.nextToken() != null) { - YamasJsonModel jsonModel = jp.readValueAs(YamasJsonModel.class); - packets.add(toMetricsPacketBuilder(jsonModel)); - } - return packets; - } catch (IOException e) { - log.log(WARNING, "Could not create metrics packet from string:\n" + jsonString, e); - return emptyList(); - } - } - - private static YamasJsonModel getStatusYamasModel(String statusMessage, int statusCode, Collection<ConsumerId> consumers) { - YamasJsonModel model = new YamasJsonModel(); - model.status_code = statusCode; - model.status_msg = statusMessage; - model.application = "yms_check_vespa"; - model.routing = ImmutableMap.of(YAMAS_ROUTING, toYamasJsonNamespaces(consumers)); - return model; - } - - private static YamasJsonModel toYamasModel(MetricsPacket packet, boolean addStatus) { - YamasJsonModel model = new YamasJsonModel(); - - if (addStatus) { - model.status_code = packet.statusCode; - model.status_msg = packet.statusMessage; - } - - model.application = packet.service.id; - model.timestamp = (packet.timestamp == 0L) ? null : packet.timestamp; - - if (packet.metrics().isEmpty()) model.metrics = null; - else { - model.metrics = packet.metrics().entrySet().stream().collect( - toLinkedMap(id2metric -> id2metric.getKey().id, - id2metric -> id2metric.getValue().doubleValue())); - } - - if (packet.dimensions().isEmpty()) model.dimensions = null; - else { - model.dimensions = packet.dimensions().entrySet().stream().collect( - toLinkedMap(id2dim -> id2dim.getKey().id, - Map.Entry::getValue)); - } - - if (packet.consumers().isEmpty()) model.routing = null; - else model.routing = ImmutableMap.of(YAMAS_ROUTING, toYamasJsonNamespaces(packet.consumers())); - - return model; - } - - private static YamasJsonModel.YamasJsonNamespace toYamasJsonNamespaces(Collection<ConsumerId> consumers) { - YamasJsonModel.YamasJsonNamespace namespaces = new YamasJsonModel.YamasJsonNamespace(); - namespaces.namespaces = consumers.stream().map(consumer -> consumer.id).collect(Collectors.toList()); - return namespaces; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java deleted file mode 100644 index fdac0521256..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.Version; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.module.SimpleModule; - -import java.io.IOException; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -/** - * Datamodel for the metricsproxy representation of multiple yamas checks. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class YamasArrayJsonModel { - @JsonProperty("metrics") - public final List<YamasJsonModel> metrics = new ArrayList<>(); - - public void add(List<YamasJsonModel> results) { - metrics.addAll(results); - } - - public void add(YamasJsonModel result) { - metrics.add(result); - } - - public void add(YamasArrayJsonModel array) { - metrics.addAll(array.metrics); - } - - /** - * Convenience method to serialize. - * <p> - * Custom floating point serializer to avoid scientifc notation - * - * @return Serialized json - */ - public String serialize() { - ObjectMapper mapper = new ObjectMapper(); - SimpleModule module = new SimpleModule("DoubleSerializer", - new Version(1, 0, 0, "", null, null)); - module.addSerializer(Double.class, new DoubleSerializer()); - mapper.registerModule(module); - - if (metrics.size() > 0) { - try { - return mapper.writeValueAsString(this); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - - return "{}"; // Backwards compatability - } - - public class DoubleSerializer extends JsonSerializer<Double> { - @Override - public void serialize(Double value, JsonGenerator jgen, - SerializerProvider provider) throws IOException, JsonProcessingException { - DecimalFormat df = new DecimalFormat("#.####", new DecimalFormatSymbols(Locale.ENGLISH)); - df.setMaximumFractionDigits(13); - jgen.writeNumber(df.format(value)); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java deleted file mode 100644 index 5fdbe9577be..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.yahoo.stream.CustomCollectors.toLinkedMap; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; - -/** - * Datamodel for Yamas execute output - * <p> - * Used to read from original yamas checks and annotate with routing information. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ "status_code", "timestamp", "application", "metrics", "dimensions", "routing", "status_msg"}) -public class YamasJsonModel { - @JsonProperty("status_code") - public Integer status_code; - @JsonProperty("status_msg") - public String status_msg; - @JsonProperty("timestamp") - public Long timestamp; - @JsonProperty("application") - public String application; - @JsonProperty("metrics") - public Map<String, Double> metrics; - @JsonProperty("dimensions") - public Map<String, String> dimensions; - @JsonProperty("routing") - public Map<String, YamasJsonNamespace> routing; - - public static class YamasJsonNamespace { - @JsonProperty("namespaces") - public List<String> namespaces; - } - - // NOTE: do not rename to 'setMetrics', as jackson will try to use it. - public void resetMetrics(List<Metric> newMetrics) { - metrics = new LinkedHashMap<>(); - newMetrics.forEach(metric -> metrics.put(metric.getName(), metric.getValue().doubleValue())); - } - - /** - * Convenience method to add targets to the routing object - * - * @param names Namespaces E.g "Vespa" - */ - public void addRouting(Set<ConsumerId> names) { - // Setup routing structure if not already existing - if (routing == null) { - routing = new HashMap<>(); - } - - if (! routing.containsKey("yamas")) { - routing.put("yamas", new YamasJsonModel.YamasJsonNamespace()); - } - YamasJsonModel.YamasJsonNamespace namespace = routing.get("yamas"); - - if (namespace.namespaces == null) { - namespace.namespaces = new ArrayList<>(); - } - - namespace.namespaces.addAll(names.stream().map(consumer -> consumer.id).collect(Collectors.toList())); - } - - /** - * Convenience method to add dimensions - */ - public void addDimensions(Map<DimensionId, String> additionalDimensions, boolean replace) { - additionalDimensions.forEach((k,v) -> { - addDimension(k.id, v, replace); - }); - } - - /** - * Convenience method to add dimensions - */ - public void addDimension(String key, String value, boolean replace) { - if (dimensions == null) { - dimensions = new HashMap<>(); - } - if (!dimensions.containsKey(key) || replace) { - dimensions.put(key, value); - } - } - - List<Metric> getMetricsList() { - if (metrics == null) return emptyList(); - - return metrics.keySet().stream() - .map(name -> new Metric(name, metrics.get(name))) - .collect(Collectors.toList()); - } - - Map<DimensionId, String> getDimensionsById() { - if (dimensions == null) return emptyMap(); - - return dimensions.keySet().stream().collect(toLinkedMap(DimensionId::toDimensionId, - name -> dimensions.get(name))); - } - - Set<ConsumerId> getYamasConsumers() { - if (routing == null || routing.get("yamas") == null) return emptySet(); - - return routing.get("yamas").namespaces.stream() - .map(ConsumerId::toConsumerId) - .collect(Collectors.toCollection(LinkedHashSet::new)); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java deleted file mode 100644 index c72f2484f8c..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -@ExportPackage -package ai.vespa.metricsproxy.metric; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java deleted file mode 100644 index e7feab9926d..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.jrt.Acceptor; -import com.yahoo.jrt.ListenFailedException; -import com.yahoo.jrt.Method; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Transport; - -import java.util.logging.Logger; - -import static com.yahoo.log.LogLevel.DEBUG; -import static java.util.logging.Level.INFO; - -/** - * Contains the connector for the rpc server, to prevent it from going down after component reconfiguration. - * This will only be recreated if the rpc port changes, which should never happen under normal circumstances. - * - * @author gjoranv - */ -public class RpcConnector extends AbstractComponent { - private static final Logger log = Logger.getLogger(RpcConnector.class.getName()); - - private final Supervisor supervisor = new Supervisor(new Transport()); - private final Acceptor acceptor; - - public RpcConnector(RpcConnectorConfig config) { - Spec spec = new Spec(config.port()); - try { - acceptor = supervisor.listen(spec); - log.log(DEBUG, "Listening on " + spec.host() + ":" + spec.port()); - } catch (ListenFailedException e) { - stop(); - log.log(INFO, "Failed listening at " + spec.host() + ":" + spec.port()); - throw new RuntimeException("Could not listen at " + spec, e); - } - } - - /** - * Adds a method. If a method with the same name already exists, it will be replaced. - * @param method The method to add. - */ - public void addMethod(Method method) { - supervisor.addMethod(method); - } - - public void stop() { - acceptor.shutdown().join(); - supervisor.transport().shutdown().join(); - } - - @Override - public void deconstruct() { - stop(); - super.deconstruct(); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java deleted file mode 100644 index e0e0e7a3f87..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; -import com.yahoo.jrt.ErrorCode; -import com.yahoo.jrt.Method; -import com.yahoo.jrt.Request; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.StringValue; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Transport; - -import java.time.Instant; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toMetricsPackets; -import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toYamasArray; -import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.log.LogLevel.DEBUG; -import static java.util.logging.Level.INFO; -import static java.util.logging.Level.WARNING; - -/** - * Rpc server for the metrics proxy. - * - * When a new object is created after reconfiguration, it will claim ownership of the methods - * in the given {@link RpcConnector}. This is ok because at the time this component is created, - * all components it depends on are already created. - * - * @author gjoranv - */ -public class RpcServer { - - private static final Logger log = Logger.getLogger(RpcServer.class.getName()); - - private static int LOG_SPENT_TIME_LIMIT = 10 * 1000; // ms. same as default client RPC timeout used in rpc_invoke - - private final VespaServices vespaServices; - private final MetricsManager metricsManager; - - public RpcServer(RpcConnector connector, VespaServices vespaServices, MetricsManager metricsManager) { - this.vespaServices = vespaServices; - this.metricsManager = metricsManager; - addMethods(connector); - log.log(DEBUG, "RPC server created"); - } - - private void addMethods(RpcConnector connector) { - // Add/replace this method first to increase likelihood of getting extra metrics and global dimensions - connector.addMethod( - new Method("setExtraMetrics", "s", "", this::setExtraMetrics) - .methodDesc("Set extra metrics that will be added to output from getMetricsForYamas.") - .paramDesc(0, "metricsJson", "The metrics in json format")); - - connector.addMethod( - new Method("getMetricsById", "s", "s", this::getMetricsById) - .methodDesc("Get Vespa metrics for the service with the given Id") - .paramDesc(0, "id", "The id of the service") - .returnDesc(0, "ret", "Vespa metrics")); - - connector.addMethod( - new Method("getServices", "", "s", this::getServices) - .methodDesc("Get Vespa services monitored by this metrics proxy") - .returnDesc(0, "ret", "Vespa metrics")); - - connector.addMethod( - new Method("getMetricsForYamas", "s", "s", this::getMetricsForYamas) - .methodDesc("Get JSON formatted Vespa metrics for a given service name or 'all'") - .paramDesc(0, "service", "The vespa service name, or 'all'") - .returnDesc(0, "ret", "Vespa metrics")); - - connector.addMethod( - new Method("getHealthMetricsForYamas", "s", "s", this::getHealthMetricsForYamas) - .methodDesc("Get JSON formatted Health check for a given service name or 'all'") - .paramDesc(0, "service", "The vespa service name") - .returnDesc(0, "ret", "Vespa metrics")); - - connector.addMethod( - new Method("getAllMetricNamesForService", "ss", "s", this::getAllMetricNamesForService) - .methodDesc("Get metric names known for service ") - .paramDesc(0, "service", "The vespa service name'") - .paramDesc(1, "consumer", "The consumer'") - .returnDesc(0, "ret", "Metric names, one metric name per line")); - } - - void getAllMetricNamesForService(Request req) { - String service = req.parameters().get(0).asString(); - ConsumerId consumer = toConsumerId(req.parameters().get(1).asString()); - withExceptionHandling(req, () -> { - String metricNames = metricsManager.getMetricNamesForServiceAndConsumer(service, consumer); - req.returnValues().add(new StringValue(metricNames)); - }); - } - - void getMetricsById(Request req) { - String id = req.parameters().get(0).asString(); - withExceptionHandling(req, () -> { - String metricsString = metricsManager.getMetricsByConfigId(id); - req.returnValues().add(new StringValue(metricsString)); - }); - } - - - void getServices(Request req) { - withExceptionHandling(req, () -> { - String servicesString = metricsManager.getAllVespaServices(); - req.returnValues().add(new StringValue(servicesString)); - }); - } - - void getMetricsForYamas(Request req) { - Instant startTime = Instant.now(); - req.detach(); - String service = req.parameters().get(0).asString(); - log.log(DEBUG, () -> "getMetricsForYamas called at " + startTime + " with argument: " + service); - List<VespaService> services = vespaServices.getMonitoringServices(service); - log.log(DEBUG, () -> "Getting metrics for services: " + mkString(services, "[", ", ", "]")); - if (services.isEmpty()) setNoServiceError(req, service); - else withExceptionHandling(req, () -> { - List<MetricsPacket> packets = metricsManager.getMetrics(services, startTime); - log.log(DEBUG,() -> "Returning metrics packets:\n" + mkString(packets, "\n")); - req.returnValues().add(new StringValue(toYamasArray(packets).serialize())); - }); - req.returnRequest(); - } - - void getHealthMetricsForYamas(Request req) { - req.detach(); - String service = req.parameters().get(0).asString(); - List<VespaService> services = vespaServices.getMonitoringServices(service); - if (services.isEmpty()) setNoServiceError(req, service); - else withExceptionHandling(req, () -> { - List<MetricsPacket> packets = metricsManager.getHealthMetrics(services); - req.returnValues().add(new StringValue(toYamasArray(packets, true).serialize())); - }); - req.returnRequest(); - } - - void setExtraMetrics(Request req) { - String metricsJson = req.parameters().get(0).asString(); - log.log(DEBUG, "setExtraMetrics called with argument: " + metricsJson); - withExceptionHandling(req, () -> metricsManager.setExtraMetrics(toMetricsPackets(metricsJson))); - } - - private static void withExceptionHandling(Request req, ThrowingRunnable runnable) { - try { - TimeTracker timeTracker = new TimeTracker(req); - runnable.run(); - timeTracker.logSpentTime(); - } catch (Exception e) { - log.log(WARNING, "Got exception when running RPC command " + req.methodName(), e); - setMethodFailedError(req, e); - } catch (Error e) { - log.log(WARNING, "Got error when running RPC command " + req.methodName(), e); - setMethodFailedError(req, e); - } catch (Throwable t) { - log.log(WARNING, "Got throwable (non-error, non-exception) when running RPC command " + req.methodName(), t); - setMethodFailedError(req, t); - } - } - - private static void setMethodFailedError(Request req, Throwable t) { - String msg = "Request failed due to internal error: " + t.getClass().getName() + ": " + t.getMessage(); - req.setError(ErrorCode.METHOD_FAILED, msg); - req.returnValues().add(new StringValue("")); - } - - private static void setNoServiceError(Request req, String serviceName) { - String msg = "No service with name '" + serviceName + "'"; - req.setError(ErrorCode.BAD_REQUEST, msg); - req.returnValues().add(new StringValue("")); - } - - - private static class TimeTracker { - private final long startTime = System.currentTimeMillis(); - private final Request request; - - private TimeTracker(Request request) { - this.request = request; - } - - long spentTime() { - return System.currentTimeMillis() - startTime; - } - - private void logSpentTime() { - Level logLevel = DEBUG; - if (spentTime() > LOG_SPENT_TIME_LIMIT) { - logLevel = INFO; - } - if (log.isLoggable(logLevel)) { - log.log(logLevel, "RPC request '" + request.methodName() + "' with parameters '" + - request.parameters() + "' took " + spentTime() + " ms"); - } - } - } - - private interface ThrowingRunnable { - void run() throws Exception; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java deleted file mode 100644 index 875b6190763..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import com.google.inject.Inject; -import com.yahoo.log.LogLevel; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - -/** - * Connects to the config sentinel and gets information like pid for the services on the node - */ -public class ConfigSentinelClient { - private final static Logger log = Logger.getLogger(ConfigSentinelClient.class.getName()); - - private final CmdClient client; - - @Inject - public ConfigSentinelClient() { - this.client = new CmdClient(); - } - - /** - * Update all services reading from config sentinel - * - * @param services The list of services - */ - synchronized void updateServiceStatuses(List<VespaService> services) { - try { - setStatus(services); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e); - } - } - - - /** - * Update status - * - * @param s The service to update the status for - */ - public synchronized void ping(VespaService s) { - List<VespaService> services = new ArrayList<>(); - services.add(s); - log.log(LogLevel.DEBUG, "Ping for service " + s); - try { - setStatus(services); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e); - } - } - - /** - * Update the status (pid check etc) - * - * @param services list of services - * @throws Exception if something went wrong - */ - protected synchronized void setStatus(List<VespaService> services) throws Exception { - InputStream in; - PrintStream out; - client.connect(); - - in = client.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(in)); - String line; - List<VespaService> updatedServices = new ArrayList<>(); - while ((line = reader.readLine()) != null) { - if (line.equals("")) { - break; - } - - VespaService s = parseServiceString(line, services); - if (s != null) { - updatedServices.add(s); - } - } - - //Check if there are services that were not found in output - //from the sentinel - for (VespaService s : services) { - if ((!s.getServiceName().equals("configserver")) && !updatedServices.contains(s)) { - log.log(LogLevel.DEBUG,"Service " + s + " is no longer found with sentinel - setting alive = false"); - s.setAlive(false); - } - } - - //Close streams - reader.close(); - client.disconnect(); - } - - protected static VespaService parseServiceString(String line, List<VespaService> services) { - String[] parts = line.split(" "); - if (parts.length < 3) - return null; - - String name = parts[0]; - int pid = -1; - String state = null; - VespaService service = null; - - for (VespaService s : services) { - if (s.getInstanceName().compareToIgnoreCase(name) == 0) { - service = s; - break; - } - } - - //Could not find this service - //nothing wrong with that as the check is invoked per line from sentinel - if (service == null) { - return service; - } - - for (int i = 1; i < parts.length; i++) { - String keyValue[] = parts[i].split("="); - - String key = keyValue[0]; - String value = keyValue[1]; - - if (key.equals("state")) { - state = value; - } else if (key.equals("pid")) { - pid = Integer.parseInt(value); - } - } - - if (state != null) { - service.setState(state); - if (pid >= 0 && "RUNNING".equals(state)) { - service.setAlive(true); - service.setPid(pid); - } else { - service.setAlive(false); - - } - } else { - service.setAlive(false); - } - return service; - } - - static class CmdClient { - Process proc; - // NOTE: hostname/port not used yet - void connect() { - String[] args = new String[]{"vespa-sentinel-cmd", "list"}; - try { - proc = Runtime.getRuntime().exec(args); - } catch (Exception e) { - log.log(LogLevel.WARNING, "could not run vespa-sentinel-cmd: "+e); - proc = null; - } - } - void disconnect() { - if (proc.isAlive()) { - proc.destroy(); - } - proc = null; - } - InputStream getInputStream() { - return (proc != null) - ? proc.getInputStream() - : new java.io.ByteArrayInputStream(new byte[0]); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java deleted file mode 100644 index 312e7d5c0c1..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -class CpuJiffies { - private int cpuId; - private long jiffies; - - CpuJiffies(String line) { - parseLine(line); - } - - private void parseLine(String line) { - String elems[]; - String cpuId; - long jiffies; - - elems = line.split("\\s+"); - cpuId = elems[0].substring(3); - if (cpuId.length() == 0) { - this.cpuId = -1; - } else { - this.cpuId = Integer.parseInt(cpuId); - } - - jiffies = 0; - for (int i = 1; i < elems.length; i++) { - jiffies += Long.parseLong(elems[i].replaceAll("[\\n\\r]+", "")); - } - - this.jiffies = jiffies; - } - - public int getCpuId() { - return cpuId; - } - - public long getTotalJiffies() { - return jiffies; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java deleted file mode 100644 index f87171a42dc..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.HealthMetric; - -/** - * Dummy class used for getting health status for a vespa service that has no HTTP service - * for getting health status - * - * @author hmusum - */ -public class DummyHealthMetricFetcher extends RemoteHealthMetricFetcher { - - /** - * @param service The service to fetch metrics from - */ - DummyHealthMetricFetcher(VespaService service) { - super(service, 0); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public HealthMetric getHealth(int fetchCount) { - if (service.isAlive()) { - return HealthMetric.getOk("Service is running - pid check only"); - } else { - return HealthMetric.getFailed("Service is not running - pid check only"); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java deleted file mode 100644 index f21d125e279..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.Metrics; - -/** - * Dummy class used for getting health status for a vespa service that has no HTTP service - * for getting metrics - * - * @author hmusum - */ -public class DummyMetricsFetcher extends RemoteMetricsFetcher { - - /** - * @param service The service to fetch metrics from - */ - DummyMetricsFetcher(VespaService service) { - super(service, 0); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public Metrics getMetrics(int fetchCount) { - return new Metrics(); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java deleted file mode 100644 index 9094ef22c20..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ /dev/null @@ -1,94 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.util.http.VespaHttpClientBuilder; -import com.yahoo.log.LogLevel; -import com.yahoo.yolean.Exceptions; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; -import org.apache.http.impl.client.CloseableHttpClient; - -import java.io.IOException; -import java.net.URI; -import java.util.logging.Logger; - -/** - * HTTP client to get metrics or health data from a service - * - * @author hmusum - * @author bjorncs - */ -public abstract class HttpMetricFetcher { - private final static Logger log = Logger.getLogger(HttpMetricFetcher.class.getPackage().getName()); - public final static String STATE_PATH = "/state/v1/"; - final static String METRICS_PATH = STATE_PATH + "metrics"; - final static String HEALTH_PATH = STATE_PATH + "health"; - // The call to apache will do 3 retries. As long as we check the services in series, we can't have this too high. - public static int CONNECTION_TIMEOUT = 5000; - private final static int SOCKET_TIMEOUT = 60000; - private final URI url; - protected final VespaService service; - private static final CloseableHttpClient httpClient = createHttpClient(); - - - /** - * @param service The service to fetch metrics from - * @param port The port to use - */ - HttpMetricFetcher(VespaService service, int port, String path) { - this.service = service; - - String u = "http://localhost:" + port + path; - this.url = URI.create(u); - log.log(LogLevel.DEBUG, "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT); - } - - String getJson() throws IOException { - log.log(LogLevel.DEBUG, "Connecting to url " + url + " for service '" + service + "'"); - return httpClient.execute(new HttpGet(url), new BasicResponseHandler()); - } - - public String toString() { - return this.getClass().getSimpleName() + " using " + url; - } - - String errMsgNoResponse(IOException e) { - return "Unable to get response from service '" + service + "': " + - Exceptions.toMessageString(e); - } - - void handleException(Exception e, String data, int timesFetched) { - logMessage("Unable to parse json '" + data + "' for service '" + service + "': " + - Exceptions.toMessageString(e), timesFetched); - } - - private void logMessage(String message, int timesFetched) { - if (service.isAlive() && timesFetched > 5) { - log.log(LogLevel.INFO, message); - } else { - log.log(LogLevel.DEBUG, message); - } - } - - void logMessageNoResponse(String message, int timesFetched) { - if (timesFetched > 5) { - log.log(LogLevel.WARNING, message); - } else { - log.log(LogLevel.INFO, message); - } - } - - private static CloseableHttpClient createHttpClient() { - return VespaHttpClientBuilder.create() - .setUserAgent("metrics-proxy-http-client") - .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(CONNECTION_TIMEOUT) - .setSocketTimeout(SOCKET_TIMEOUT) - .build()) - .build(); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java deleted file mode 100644 index 503f582a827..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ /dev/null @@ -1,77 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.HealthMetric; -import com.yahoo.log.LogLevel; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.IOException; -import java.util.logging.Logger; - -/** - * Fetch health status for a given vespa service - * - * @author Jo Kristian Bergum - */ -public class RemoteHealthMetricFetcher extends HttpMetricFetcher { - - private final static Logger log = Logger.getLogger(RemoteHealthMetricFetcher.class.getPackage().getName()); - - /** - * @param service The service to fetch metrics from - * @param port The port to use - */ - public RemoteHealthMetricFetcher(VespaService service, int port) { - super(service, port, HEALTH_PATH); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public HealthMetric getHealth(int fetchCount) { - String data = "{}"; - try { - data = getJson(); - } catch (IOException e) { - logMessageNoResponse(errMsgNoResponse(e), fetchCount); - } - return createHealthMetrics(data, fetchCount); - } - - /** - * Connect to remote service over http and fetch metrics - */ - HealthMetric createHealthMetrics(String data, int fetchCount) { - HealthMetric healthMetric = HealthMetric.getFailed("Failed fetching status page for service"); - try { - healthMetric = parse(data); - } catch (Exception e) { - handleException(e, data, fetchCount); - } - return healthMetric; - } - - private HealthMetric parse(String data) { - if (data == null || data.isEmpty()) { - return HealthMetric.getFailed("Empty response from status page"); - } - try { - JSONObject o = new JSONObject(data); - JSONObject status = o.getJSONObject("status"); - String code = status.getString("code"); - String message = ""; - if (status.has("message")) { - message = status.getString("message"); - } - return HealthMetric.get(code, message); - - } catch (JSONException e) { - log.log(LogLevel.DEBUG, "Failed to parse json response from metrics page:" + e + ":" + data); - return HealthMetric.getFailed("Not able to parse json from status page"); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java deleted file mode 100644 index a606ec7d8cd..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ /dev/null @@ -1,129 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; - -/** - * Fetch metrics for a given vespa service - * - * @author Jo Kristian Bergum - */ -public class RemoteMetricsFetcher extends HttpMetricFetcher { - /** - * @param service The service to fetch metrics from - * @param port The port to use - */ - RemoteMetricsFetcher(VespaService service, int port) { - super(service, port, METRICS_PATH); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public Metrics getMetrics(int fetchCount) { - String data = "{}"; - try { - data = getJson(); - } catch (IOException e) { - logMessageNoResponse(errMsgNoResponse(e), fetchCount); - } - - return createMetrics(data, fetchCount); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public Metrics createMetrics(String data, int fetchCount) { - Metrics remoteMetrics = new Metrics(); - try { - remoteMetrics = parse(data); - } catch (Exception e) { - handleException(e, data, fetchCount); - } - - return remoteMetrics; - } - - Metrics parse(String data) throws JSONException { - JSONObject o = new JSONObject(data); - if (!(o.has("metrics"))) { - return new Metrics(); //empty - } - - JSONObject metrics = o.getJSONObject("metrics"); - JSONArray values; - long timestamp; - - try { - JSONObject snapshot = metrics.getJSONObject("snapshot"); - timestamp = (long) snapshot.getDouble("to"); - values = metrics.getJSONArray("values"); - } catch (JSONException e) { - // snapshot might not have been produced. Do not throw exception into log - return new Metrics(); - } - long now = System.currentTimeMillis() / 1000; - timestamp = Metric.adjustTime(timestamp, now); - Metrics m = new Metrics(timestamp); - - Map<DimensionId, String> noDims = Collections.emptyMap(); - Map<String, Map<DimensionId, String>> uniqueDimensions = new HashMap<>(); - for (int i = 0; i < values.length(); i++) { - JSONObject metric = values.getJSONObject(i); - String name = metric.getString("name"); - String description = ""; - - if (metric.has("description")) { - description = metric.getString("description"); - } - - Map<DimensionId, String> dim = noDims; - if (metric.has("dimensions")) { - JSONObject dimensions = metric.getJSONObject("dimensions"); - StringBuilder sb = new StringBuilder(); - for (Iterator<?> it = dimensions.keys(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.getString(k); - sb.append(toDimensionId(k)).append(v); - } - if ( ! uniqueDimensions.containsKey(sb.toString())) { - dim = new HashMap<>(); - for (Iterator<?> it = dimensions.keys(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.getString(k); - dim.put(toDimensionId(k), v); - } - uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim)); - } - dim = uniqueDimensions.get(sb.toString()); - } - - JSONObject aggregates = metric.getJSONObject("values"); - for (Iterator<?> it = aggregates.keys(); it.hasNext(); ) { - String aggregator = (String) it.next(); - Number value = (Number) aggregates.get(aggregator); - StringBuilder metricName = (new StringBuilder()).append(name).append(".").append(aggregator); - m.add(new Metric(metricName.toString(), value, timestamp, dim, description)); - } - } - - return m; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java deleted file mode 100644 index 810eb5fb908..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java +++ /dev/null @@ -1,14 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import java.util.List; - -/** - * @author Unknown - */ -public interface ServiceListener { - void setServices(List<VespaService> services); -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java deleted file mode 100644 index 9f6614668a5..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import com.yahoo.log.LogLevel; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Logger; - -/** - * Class to get data from the system and update the services at given intervals. - * TODO: rewrite to use ScheduledExecutorService or just call poll() directly. - * - * @author Eirik Nygaard - */ -public class SystemPoller implements ServiceListener { - final private static Logger log = Logger.getLogger(SystemPoller.class.getPackage().getName()); - - private final int pollingIntervalSecs; - private volatile List<VespaService> services; - - private final int memoryTypeVirtual = 0; - private final int memoryTypeResident = 1; - private final Map<VespaService, Long> lastCpuJiffiesMetrics = new ConcurrentHashMap<>(); - private final Timer systemPollTimer; - - private long lastTotalCpuJiffies = -1; - - public SystemPoller(List<VespaService> services, int pollingIntervalSecs) { - this.services = services; - this.pollingIntervalSecs = pollingIntervalSecs; - systemPollTimer = new Timer("systemPollTimer", true); - } - - @Override - public void setServices(List<VespaService> services) { - log.log(LogLevel.DEBUG, "Setting services in SystemPoller to: " + services); - this.services = services; - } - - void stop() { - systemPollTimer.cancel(); - } - - /** - * Return memory usage for a given process, both resident and virtual is - * returned. - * - * @param service The instance to get memory usage for - * @return array[0] = memoryResident, array[1] = memoryVirtual (kB units) - */ - long[] getMemoryUsage(VespaService service) { - long size[] = new long[2]; - BufferedReader br; - int pid = service.getPid(); - - size[0] = 0; - size[1] = 0; - try { - br = new BufferedReader(new FileReader("/proc/" + pid + "/smaps")); - } catch (FileNotFoundException ex) { - markDead(service); - return size; - } - String line; - try { - while ((line = br.readLine()) != null) { - String[] elems = line.split("\\s+"); - /* Memory size is given in kB - convert to bytes by multiply with 1024*/ - if (line.startsWith("Rss:")) { - size[memoryTypeResident] += Long.parseLong(elems[1]) * 1024; - } else if (line.startsWith("Size:")) { - size[memoryTypeVirtual] += Long.parseLong(elems[1]) * 1024; - } - } - - br.close(); - } catch (IOException ex) { - log.log(LogLevel.DEBUG, "Unable to read line from smaps file", ex); - return size; - } - - return size; - } - - /** - * Mark a service as dead. - * - * @param service The service to mark as dead. - */ - private static void markDead(VespaService service) { - service.setAlive(false); - } - - /** - * Poll services for system metrics - */ - void poll() { - long startTime = System.currentTimeMillis(); - boolean someAlive = false; - - /* Don't do any work if there are no known services */ - if (services.isEmpty()) { - schedule(); - return; - } - - log.log(LogLevel.DEBUG, "Monitoring system metrics for " + services.size() + " services"); - - long sysJiffies = getNormalizedSystemJiffies(); - for (VespaService s : services) { - - - if(s.isAlive()) { - someAlive = true; - } - - Metrics metrics = new Metrics(); - log.log(LogLevel.DEBUG, "Current size of system metrics for service " + s + " is " + metrics.size()); - - long[] size = getMemoryUsage(s); - log.log(LogLevel.DEBUG, "Updating memory metric for service " + s); - - metrics.add(new Metric("memory_virt", size[memoryTypeVirtual], startTime / 1000)); - metrics.add(new Metric("memory_rss", size[memoryTypeResident], startTime / 1000)); - - long procJiffies = getPidJiffies(s); - if (lastTotalCpuJiffies >= 0 && lastCpuJiffiesMetrics.containsKey(s)) { - long last = lastCpuJiffiesMetrics.get(s); - long diff = procJiffies - last; - - if (diff >= 0) { - metrics.add(new Metric("cpu", 100 * ((double) diff) / (sysJiffies - lastTotalCpuJiffies), startTime / 1000)); - } - } - lastCpuJiffiesMetrics.put(s, procJiffies); - s.setSystemMetrics(metrics); - } - - lastTotalCpuJiffies = sysJiffies; - - // If none of the services were alive, reschedule in a short time - if (!someAlive) { - reschedule(System.currentTimeMillis() - startTime); - } else { - schedule(); - } - } - - long getPidJiffies(VespaService service) { - BufferedReader in; - String line; - String[] elems; - int pid = service.getPid(); - - try { - in = new BufferedReader(new FileReader("/proc/" + pid + "/stat")); - } catch (FileNotFoundException ex) { - log.log(LogLevel.DEBUG, "Unable to find pid in proc directory " + pid); - service.setAlive(false); - return 0; - } - - try { - line = in.readLine(); - in.close(); - } catch (IOException ex) { - log.log(LogLevel.DEBUG, "Unable to read line from process stat file", ex); - return 0; - } - - elems = line.split(" "); - - /* Add user mode and kernel mode jiffies for the given process */ - return Long.parseLong(elems[13]) + Long.parseLong(elems[14]); - } - - long getNormalizedSystemJiffies() { - BufferedReader in; - String line; - ArrayList<CpuJiffies> jiffies = new ArrayList<>(); - CpuJiffies total = null; - - try { - in = new BufferedReader(new FileReader("/proc/stat")); - } catch (FileNotFoundException ex) { - log.log(LogLevel.ERROR, "Unable to open stat file", ex); - return 0; - } - try { - while ((line = in.readLine()) != null) { - if (line.startsWith("cpu ")) { - total = new CpuJiffies(line); - } else if (line.startsWith("cpu")) { - jiffies.add(new CpuJiffies(line)); - } - } - - in.close(); - } catch (IOException ex) { - log.log(LogLevel.ERROR, "Unable to read line from stat file", ex); - return 0; - } - - /* Normalize so that a process that uses an entire CPU core will get 100% util */ - if (total != null) { - return total.getTotalJiffies() / jiffies.size(); - } else { - return 0; - } - } - - private void schedule(long time) { - try { - systemPollTimer.schedule(new PollTask(this), time); - } catch(IllegalStateException e){ - log.info("Tried to schedule task, but timer was already shut down."); - } - } - - public void schedule() { - schedule(pollingIntervalSecs * 1000); - } - - private void reschedule(long skew) { - long sleep = (pollingIntervalSecs * 1000) - skew; - - // Don't sleep less than 1 min - sleep = Math.max(60 * 1000, sleep); - schedule(sleep); - } - - - private static class PollTask extends TimerTask { - private final SystemPoller poller; - - PollTask(SystemPoller poller) { - this.poller = poller; - } - - @Override - public void run() { - poller.poll(); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java deleted file mode 100644 index dea7b2b4809..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.core.MonitoringConfig; -import com.yahoo.container.di.componentgraph.Provider; - -/** - * @author gjoranv - */ -public class SystemPollerProvider implements Provider<SystemPoller> { - - private final SystemPoller poller; - - /** - * @param services The list of VespaService instances to monitor for System metrics - * @param monitoringConfig The interval in seconds between each polling. - */ - public SystemPollerProvider (VespaServices services, MonitoringConfig monitoringConfig) { - poller = new SystemPoller(services.getVespaServices(), 60 * monitoringConfig.intervalMinutes()); - poller.poll(); - } - - public void deconstruct() { - poller.stop(); - } - - public SystemPoller get() { - return poller; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java deleted file mode 100644 index d3f674176b3..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - - -/** - * Represents a Vespa service - * - * @author jobergum - */ -public class VespaService implements Comparable<VespaService> { - - private static final Map<DimensionId, String> EMPTY_DIMENSIONS = Collections.emptyMap(); - private static final String DEFAULT_MONITORING_PREFIX = "vespa"; - public static final String SEPARATOR = "."; - - private final String instanceName; - private final String configId; - private final String serviceName; - private final String monitoringPrefix; - private final Map<DimensionId, String> dimensions; - - - private volatile int pid = -1; - private volatile String state = "UNKNOWN"; - - // Used to keep the last polled system metrics for service - private Metrics systemMetrics; - - private final int statePort; - - private final RemoteHealthMetricFetcher remoteHealthMetricFetcher; - private final RemoteMetricsFetcher remoteMetricsFetcher; - - private boolean isAlive; - - // Used to keep track of log level when health or metrics requests fail - private AtomicInteger metricsFetchCount = new AtomicInteger(0); - private AtomicInteger healthFetchCount = new AtomicInteger(0); - - - public static VespaService create(String name, String id, int statePort) { - return create(name,id, statePort, DEFAULT_MONITORING_PREFIX, EMPTY_DIMENSIONS); - } - - public static VespaService create(String name, String id, int statePort, String monitoringName, Map<DimensionId, String> dimensions) { - String serviceName = name.replaceAll("\\d*$", ""); - return new VespaService(serviceName, name, id, statePort, monitoringName, dimensions); - } - - VespaService(String serviceName, String configId) { - this(serviceName, serviceName, configId); - } - - VespaService(String serviceName, String instanceName, String configId) { - this(serviceName, instanceName, configId, -1, DEFAULT_MONITORING_PREFIX, EMPTY_DIMENSIONS); - } - - private VespaService(String serviceName, String instanceName, String configId, - int statePort, String monitoringPrefix, - Map<DimensionId, String> dimensions) { - this.serviceName = serviceName; - this.instanceName = instanceName; - this.monitoringPrefix = monitoringPrefix; - this.configId = configId; - this.statePort = statePort; - this.dimensions = dimensions; - this.systemMetrics = new Metrics(); - this.isAlive = false; - this.remoteMetricsFetcher = (this.statePort> 0) ? new RemoteMetricsFetcher(this, this.statePort) : new DummyMetricsFetcher(this); - this.remoteHealthMetricFetcher = (this.statePort > 0) ? new RemoteHealthMetricFetcher(this, this.statePort) : new DummyHealthMetricFetcher(this); - } - - /** - * The name used for this service in the monitoring system: - * monitoring-system-name.serviceName - */ - public String getMonitoringName() { - return monitoringPrefix + SEPARATOR + serviceName; - } - - @Override - public int compareTo(VespaService other) { - return this.getInstanceName().compareTo(other.getInstanceName()); - } - - /** - * Get the service name/type. E.g 'searchnode', but not 'searchnode2' - * - * @return the service name - */ - public String getServiceName() { - return this.serviceName; - } - - /** - * Get the instance name. E.g searchnode2 - * - * @return the instance service name - */ - public String getInstanceName() { - return this.instanceName; - } - - public Map<DimensionId, String> getDimensions() { - return dimensions; - } - - /** - * @return The health of this service - */ - public HealthMetric getHealth() { - HealthMetric healthMetric = remoteHealthMetricFetcher.getHealth(healthFetchCount.get()); - healthFetchCount.getAndIncrement(); - return healthMetric; - } - - /** - * Gets the system metrics for this service - * - * @return System metrics - */ - public synchronized Metrics getSystemMetrics() { - return this.systemMetrics; - } - - /** - * Get the Metrics registered for this service. Metrics are fetched over HTTP - * if a metric http port has been defined, otherwise from log file - * - * @return the non-system metrics - */ - public Metrics getMetrics() { - Metrics remoteMetrics = remoteMetricsFetcher.getMetrics(metricsFetchCount.get()); - metricsFetchCount.getAndIncrement(); - return remoteMetrics; - } - - /** - * Gets the config id of this service - * - * @return the config id - */ - public String getConfigId() { - return configId; - } - - /** - * The current pid of this service - * - * @return The pid - */ - public int getPid() { - return this.pid; - } - - /** - * update the pid of this service - * - * @param pid The pid that this service runs as - */ - public void setPid(int pid) { - this.pid = pid; - } - - /** - * Get the string representation of the state of this service - * - * @return string representing the state of this service - obtained from config-sentinel - */ - public String getState() { - return state; - } - - /** - * Update the state of this service - * - * @param state the new state - */ - public void setState(String state) { - this.state = state; - } - - /** - * Check if this pid/service is running - * - * @return true if the service is alive (e.g the pid is running) - */ - public boolean isAlive() { - return (isAlive && (pid >= 0)); - } - - @Override - public String toString() { - return instanceName + ":" + pid + ":" + state + ":" + configId; - } - - public void setAlive(boolean alive) { - this.isAlive = alive; - } - - public synchronized void setSystemMetrics(Metrics systemMetrics) { - this.systemMetrics = systemMetrics; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java deleted file mode 100644 index 2668c158ed6..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.core.MonitoringConfig; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.service.VespaServicesConfig.Service; -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static com.yahoo.log.LogLevel.DEBUG; - -/** - * Creates representations for the Vespa services running on the node, - * and provides methods for updating and getting them. - * - * @author gjoranv - */ -public class VespaServices { - private static final Logger log = Logger.getLogger(VespaServices.class.getName()); - - public static final String ALL_SERVICES = "all"; - - private final ConfigSentinelClient sentinel; - private final List<VespaService> services; - - @Inject - public VespaServices(VespaServicesConfig config, MonitoringConfig monitoringConfig, ConfigSentinelClient sentinel) { - this.services = createServices(config, monitoringConfig.systemName()); - this.sentinel = sentinel; - } - - @VisibleForTesting - public VespaServices(List<VespaService> services) { - this.services = services; - sentinel = null; - } - - private List<VespaService> createServices(VespaServicesConfig servicesConfig, String monitoringSystemName) { - List<VespaService> services = new ArrayList<>(); - for (Service s : servicesConfig.service()) { - log.log(DEBUG, "Re-configuring service " + s.name()); - VespaService vespaService = VespaService.create(s.name(), s.configId(), s.healthport(), monitoringSystemName, - createServiceDimensions(s)); - services.add(vespaService); - } - log.log(DEBUG, "Created new services: " + services.size()); - updateServices(services); - return services; - } - - /** - * Sets 'alive=false' for services that are no longer running. - * Note that the status is updated in-place for the given services. - */ - public void updateServices(List<VespaService> services) { - if (sentinel != null) { - log.log(DEBUG, "Updating services "); - sentinel.updateServiceStatuses(services); - } - } - - /** - * Get all known vespa services - * - * @return A list of VespaService objects - */ - public List<VespaService> getVespaServices() { - return Collections.unmodifiableList(services); - } - - /** - * @param id The configid - * @return A list with size 1 as there should only be one service with the given configid - */ - public List<VespaService> getInstancesById(String id) { - List<VespaService> myServices = new ArrayList<>(); - for (VespaService s : services) { - if (s.getConfigId().equals(id)) { - myServices.add(s); - } - } - - return myServices; - } - - /** - * Get services matching pattern for the name used in the monitoring system. - * - * @param service name in monitoring system + service name, without index, e.g: vespa.container - * @return A list of VespaServices - */ - public List<VespaService> getMonitoringServices(String service) { - if (service.equalsIgnoreCase(ALL_SERVICES)) - return services; - - List<VespaService> myServices = new ArrayList<>(); - for (VespaService s : services) { - log.log(DEBUG, () -> "getMonitoringServices. service=" + service + ", checking against " + s + ", which has monitoring name " + s.getMonitoringName()); - if (s.getMonitoringName().equalsIgnoreCase(service)) { - myServices.add(s); - } - } - - return myServices; - } - - private static Map<DimensionId, String> createServiceDimensions(Service service) { - return service.dimension().stream().collect( - toUnmodifiableLinkedMap(dim -> toDimensionId(dim.key()), Service.Dimension::value)); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java deleted file mode 100644 index b478cdf8b5b..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -@ExportPackage -package ai.vespa.metricsproxy.service; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def b/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def deleted file mode 100644 index 41b384193e5..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.metric.dimensions - -# Dimensions based on application properties -dimensions{} string diff --git a/metrics-proxy/src/main/resources/configdefinitions/consumers.def b/metrics-proxy/src/main/resources/configdefinitions/consumers.def deleted file mode 100644 index a1828b1320c..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/consumers.def +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.core - -# Consumers with metric definitions -consumer[].name string default="" -consumer[].metric[].name string -consumer[].metric[].description string default="" -consumer[].metric[].outputname string -consumer[].metric[].dimension[].key string -consumer[].metric[].dimension[].value string diff --git a/metrics-proxy/src/main/resources/configdefinitions/monitoring.def b/metrics-proxy/src/main/resources/configdefinitions/monitoring.def deleted file mode 100644 index 6fe24c86394..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/monitoring.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.core - -# The rate at which metrics are passed to the monitoring system. Currently (Apr 2019) only used by SystemPoller. -intervalMinutes int default=5 - -# The name used for this application in the monitoring system -systemName string default=vespa diff --git a/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def b/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def deleted file mode 100644 index 917cd165a9f..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.metric.dimensions - -# Dimensions based on node properties -dimensions{} string diff --git a/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def b/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def deleted file mode 100644 index 7a8450780de..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.rpc - -port int diff --git a/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def b/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def deleted file mode 100644 index 6fdacfffd42..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.service - -# Services with service id and the http port for the metrics page -service[].configId string -service[].name string default="" -service[].port int default=-1 -service[].healthport int default=-1 -service[].dimension[].key string -service[].dimension[].value string diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java deleted file mode 100644 index 5997f85a46f..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.stream.Collectors; - -/** - * @author gjoranv - */ -public class TestUtil { - - public static String getContents(String filename) { - InputStream in = TestUtil.class.getClassLoader().getResourceAsStream(filename); - if (in == null) { - throw new RuntimeException("File not found: " + filename); - } - return new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining("\n")); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java deleted file mode 100644 index 81634323269..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.core; - -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.service.DummyService; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; -import com.google.common.collect.ImmutableList; -import org.junit.Before; -import org.junit.Test; - -import java.time.Instant; -import java.util.List; -import java.util.Map; - -import static ai.vespa.metricsproxy.core.MetricsManager.VESPA_VERSION; -import static ai.vespa.metricsproxy.core.VespaMetrics.METRIC_TYPE_DIMENSION_ID; -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; -import static ai.vespa.metricsproxy.metric.ExternalMetrics.ROLE_DIMENSION; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - */ -public class MetricsManagerTest { - - private MetricsManager metricsManager; - - private static final String SERVICE_0_ID = "dummy/id/0"; - private static final String SERVICE_1_ID = "dummy/id/1"; - - private static final List<VespaService> testServices = ImmutableList.of( - new DummyService(0, SERVICE_0_ID), - new DummyService(1, SERVICE_1_ID)); - - private static final String WHITELISTED_METRIC_ID = "whitelisted"; - - @Before - public void setupMetricsManager() { - metricsManager = getMetricsManager(); - } - - @Test - public void each_service_gets_separate_metrics_packets() { - List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(2)); - - assertThat(packets.get(0).dimensions().get(toDimensionId("instance")), is("dummy0")); - assertThat(packets.get(0).metrics().get(toMetricId("c.test")), is(1.0)); - assertThat(packets.get(0).metrics().get(toMetricId("val")), is(1.05)); - - assertThat(packets.get(1).dimensions().get(toDimensionId("instance")), is("dummy1")); - assertThat(packets.get(1).metrics().get(toMetricId("c.test")), is(6.0)); - assertThat(packets.get(1).metrics().get(toMetricId("val")), is(2.35)); - } - - @Test - public void verify_expected_output_from_getMetricsById() { - String dummy0Metrics = metricsManager.getMetricsByConfigId("dummy/id/0"); - assertThat(dummy0Metrics, containsString("'dummy.id.0'.val=1.050")); - assertThat(dummy0Metrics, containsString("'dummy.id.0'.c_test=1")); - - String dummy1Metrics = metricsManager.getMetricsByConfigId("dummy/id/1"); - assertThat(dummy1Metrics, containsString("'dummy.id.1'.val=2.350")); - assertThat(dummy1Metrics, containsString("'dummy.id.1'.c_test=6")); - } - - @Test - public void getServices_returns_service_types() { - assertThat(metricsManager.getAllVespaServices(), is("dummy")); - } - - @Test - public void global_dimensions_are_added_but_do_not_override_metric_dimensions() { - List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertEquals(2, packets.size()); - assertGlobalDimensions(packets.get(0).dimensions()); - assertGlobalDimensions(packets.get(1).dimensions()); - } - - private void assertGlobalDimensions(Map<DimensionId, String> dimensions) { - assertTrue(dimensions.containsKey(VESPA_VERSION)); - assertEquals("value", dimensions.get(toDimensionId("global"))); - assertEquals("metric-dim", dimensions.get(toDimensionId("dim0"))); - } - - - @Test - public void system_metrics_are_added() { - VespaService service0 = testServices.get(0); - Metrics oldSystemMetrics = service0.getSystemMetrics(); - - service0.getSystemMetrics().add(new Metric("cpu", 1)); - - List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertEquals(3, packets.size()); - - MetricsPacket systemPacket = packets.get(0); // system metrics are added before other metrics - assertThat(systemPacket.metrics().get(toMetricId("cpu")), is(1.0)); - assertThat(systemPacket.dimensions().get(toDimensionId("metrictype")), is("system")); - - service0.setSystemMetrics(oldSystemMetrics); - } - - @Test - public void extra_metrics_packets_containing_whitelisted_metrics_are_added() { - metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))))); - - List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(3)); - } - - @Test - public void extra_metrics_packets_without_whitelisted_metrics_are_not_added() { - metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of(new Metric("not-whitelisted", 0))))); - - List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(2)); - } - - @Test - public void extra_dimensions_are_added_to_metrics_packets_that_do_not_have_those_dimensions() { - metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))) - .putDimension(ROLE_DIMENSION, "role from extraMetrics"))); - - List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - for (MetricsPacket packet : packets) { - assertThat(packet.dimensions().get(ROLE_DIMENSION), is("role from extraMetrics")); - } - } - - @Test - public void extra_dimensions_do_not_overwrite_existing_dimension_values() { - metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))) - .putDimension(METRIC_TYPE_DIMENSION_ID, "from extraMetrics"))); - - List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.get(0).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard")); - assertThat(packets.get(1).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard")); - assertThat(packets.get(2).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("from extraMetrics")); - } - - @Test - public void timestamp_is_adjusted_when_metric_is_less_than_one_minute_younger_than_start_time() { - Instant START_TIME = Instant.ofEpochSecond(0); - Instant METRIC_TIME = Instant.ofEpochSecond(59); - assertEquals(START_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME)); - } - - @Test - public void timestamp_is_adjusted_when_metric_is_less_than_one_minute_older_than_start_time() { - Instant START_TIME = Instant.ofEpochSecond(59); - Instant METRIC_TIME = Instant.ofEpochSecond(0); - assertEquals(START_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME)); - } - - @Test - public void timestamp_is_not_adjusted_when_metric_is_at_least_one_minute_younger_than_start_time() { - Instant START_TIME = Instant.ofEpochSecond(0); - Instant METRIC_TIME = Instant.ofEpochSecond(60); - assertEquals(METRIC_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME)); - } - - @Test - public void timestamp_is_not_adjusted_when_metric_is_at_least_one_minute_older_than_start_time() { - Instant START_TIME = Instant.ofEpochSecond(60); - Instant METRIC_TIME = Instant.ofEpochSecond(0); - assertEquals(METRIC_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME)); - } - - private Instant getAdjustedTimestamp(Instant startTime, Instant metricTime) { - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId("foo")) - .timestamp(metricTime.getEpochSecond()); - return MetricsManager.adjustTimestamp(builder, startTime).getTimestamp(); - } - - private MetricsManager getMetricsManager() { - VespaServices vespaServices = new VespaServices(testServices); - MetricsConsumers consumers = getMetricsConsumers(); - VespaMetrics metrics = new VespaMetrics(consumers, vespaServices); - - return new MetricsManager(vespaServices, metrics, new ExternalMetrics(consumers), - getApplicationDimensions(),getNodeDimensions()); - } - - private static MetricsConsumers getMetricsConsumers() { - Consumer.Metric.Dimension.Builder metricDimension = new Consumer.Metric.Dimension.Builder() - .key("dim0").value("metric-dim"); - - return new MetricsConsumers(new ConsumersConfig.Builder() - .consumer(new Consumer.Builder() - .name(VESPA_CONSUMER_ID.id) - .metric(new Consumer.Metric.Builder() - .name(WHITELISTED_METRIC_ID) - .outputname(WHITELISTED_METRIC_ID)) - .metric(new Consumer.Metric.Builder() - .name(DummyService.METRIC_1) - .outputname(DummyService.METRIC_1) - .dimension(metricDimension)) - .metric(new Consumer.Metric.Builder() - .name(DummyService.METRIC_2) - .outputname(DummyService.METRIC_2) - .dimension(metricDimension))) - .build()); - } - - private ApplicationDimensions getApplicationDimensions() { - return new ApplicationDimensions(new ApplicationDimensionsConfig.Builder() - .dimensions("global", "value").build()); - } - - private NodeDimensions getNodeDimensions() { - return new NodeDimensions(new NodeDimensionsConfig.Builder() - .dimensions("dim0", "should not override metric dim").build()); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java deleted file mode 100644 index 11c271d46e4..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import com.google.common.collect.ImmutableList; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID; -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.junit.Assert.assertEquals; - -/** - * @author gjoranv - */ -public class ExternalMetricsTest { - private static final ConsumerId CUSTOM_CONSUMER_1 = toConsumerId("consumer-1"); - private static final ConsumerId CUSTOM_CONSUMER_2 = toConsumerId("consumer-2"); - - @Test - public void extra_metrics_are_added() { - MetricsConsumers noConsumers = new MetricsConsumers(new ConsumersConfig.Builder().build()); - ExternalMetrics externalMetrics = new ExternalMetrics(noConsumers); - - externalMetrics.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")))); - - List<MetricsPacket.Builder> packets = externalMetrics.getMetrics(); - assertEquals(1, packets.size()); - } - - @Test - public void service_id_is_set_to_vespa_node_id() { - MetricsConsumers noConsumers = new MetricsConsumers(new ConsumersConfig.Builder().build()); - ExternalMetrics externalMetrics = new ExternalMetrics(noConsumers); - externalMetrics.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("replace_with_vespa_node_id")))); - - List<MetricsPacket.Builder> packets = externalMetrics.getMetrics(); - assertEquals(1, packets.size()); - assertEquals(VESPA_NODE_SERVICE_ID, packets.get(0).build().service); - } - - @Test - public void custom_consumers_are_added() { - ConsumersConfig consumersConfig = new ConsumersConfig.Builder() - .consumer(new ConsumersConfig.Consumer.Builder().name(CUSTOM_CONSUMER_1.id)) - .consumer(new ConsumersConfig.Consumer.Builder().name(CUSTOM_CONSUMER_2.id)) - .build(); - MetricsConsumers consumers = new MetricsConsumers(consumersConfig); - ExternalMetrics externalMetrics = new ExternalMetrics(consumers); - - externalMetrics.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")))); - - List<MetricsPacket.Builder> packets = externalMetrics.getMetrics(); - assertEquals(1, packets.size()); - - List<ConsumerId> consumerIds = packets.get(0).build().consumers(); - assertEquals(2, consumerIds.size()); - assertEquals(CUSTOM_CONSUMER_1, consumerIds.get(0)); - assertEquals(CUSTOM_CONSUMER_2, consumerIds.get(1)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java deleted file mode 100644 index b9e6377c27b..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.service.DummyService; -import ai.vespa.metricsproxy.service.VespaService; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknowm - */ -public class MetricsTest { - - @Test - public void testIterator() { - Metrics m = new Metrics(); - long now = System.currentTimeMillis() / 1000; - m.add(new Metric("a", 1, now)); - m.add(new Metric("b", 2.5, now)); - - //should expire after 0 seconds - m.add(new Metric("c", 2, now)); - - Map<String, Number> map = new HashMap<>(); - - for (Metric metric: m.getMetrics()) { - String k = metric.getName(); - - assertThat(map.containsKey(k), is(false)); - map.put(k, metric.getValue()); - } - - assertThat(map.get("a").intValue(), is(1)); - assertThat(map.get("b").doubleValue(), is(2.5)); - } - - @Test - public void testBasicMetric() { - Metrics m = new Metrics(); - m.add(new Metric("count", 1, System.currentTimeMillis() / 1000)); - assertThat(m.get("count").intValue(), is(1)); - } - - @Test - public void testHealthMetric() { - HealthMetric m = HealthMetric.get(null, null); - assertThat(m.isOk(), is(false)); - m = HealthMetric.get("up", "test message"); - assertThat(m.isOk(), is(true)); - assertThat(m.getMessage(), is("test message")); - m = HealthMetric.get("ok", "test message"); - assertThat(m.isOk(), is(true)); - assertThat(m.getMessage(), is("test message")); - - m = HealthMetric.get("bad", "test message"); - assertThat(m.isOk(), is(false)); - assertThat(m.getStatus(), is("bad")); - } - - @Test - public void testMetricFormatter() { - MetricsFormatter formatter = new MetricsFormatter(false, false); - VespaService service = new DummyService(0, "config.id"); - String data = formatter.format(service, "key", 1); - assertThat(data, is("'config.id'.key=1")); - - formatter = new MetricsFormatter(true, false); - data = formatter.format(service, "key", 1); - assertThat(data, is("dummy.'config.id'.key=1")); - - - formatter = new MetricsFormatter(true, true); - data = formatter.format(service, "key", 1); - assertThat(data, is("dummy.config.'id'.key=1")); - - formatter = new MetricsFormatter(false, true); - data = formatter.format(service, "key", 1); - assertThat(data, is("config.'id'.key=1")); - } - - @Test - public void testTimeAdjustment() { - assertThat(Metric.adjustTime(0L, 0L), is(0L)); - assertThat(Metric.adjustTime(59L, 59L), is(59L)); - assertThat(Metric.adjustTime(60L, 60L), is(60L)); - assertThat(Metric.adjustTime(59L, 60L), is(60L)); - assertThat(Metric.adjustTime(60L, 59L), is(60L)); - assertThat(Metric.adjustTime(59L, 61L), is(59L)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java deleted file mode 100644 index d522a56a9ac..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import ai.vespa.metricsproxy.metric.Metric; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import org.junit.Test; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author gjoranv - */ -public class MetricsPacketTest { - - @Test - public void service_cannot_be_null() { - try { - MetricsPacket packet = new MetricsPacket.Builder(null) - .statusCode(0) - .statusMessage("") - .timestamp(0L) - .build(); - fail("Expected exception due to null service."); - } catch (Exception e) { - assertEquals("Service cannot be null.", e.getMessage()); - } - } - - @Test - public void consumers_are_always_distinct() { - ConsumerId DUPLICATE_CONSUMER = toConsumerId("duplicateConsumer"); - - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .statusCode(0) - .statusMessage("") - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) - .build(); - assertEquals(1, packet.consumers().size()); - } - - @Test - public void builder_can_retain_subset_of_metrics() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of( - new Metric("remove", 1), - new Metric("keep", 2))) - .retainMetrics(ImmutableSet.of(toMetricId("keep"), toMetricId("non-existent"))) - .build(); - - assertFalse("should not contain 'remove'", packet.metrics().containsKey(toMetricId("remove"))); - assertTrue("should contain 'keep'", packet.metrics().containsKey(toMetricId("keep"))); - assertFalse("should not contain 'non-existent'", packet.metrics().containsKey(toMetricId("non-existent"))); - } - - @Test - public void builder_applies_output_names() { - String ONE = "one"; - String TWO = "two"; - String THREE = "three"; - String NON_EXISTENT = "non-existent"; - MetricId ONE_ID = toMetricId(ONE); - MetricId TWO_ID = toMetricId(TWO); - MetricId THREE_ID = toMetricId(THREE); - MetricId NON_EXISTENT_ID = toMetricId(NON_EXISTENT); - - Map<MetricId, List<String>> outputNamesById = ImmutableMap.of( - toMetricId(ONE), ImmutableList.of(ONE), - toMetricId(TWO), ImmutableList.of(TWO, "dos"), - toMetricId(THREE), ImmutableList.of("3"), - toMetricId(NON_EXISTENT), ImmutableList.of(NON_EXISTENT)); - - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of( - new Metric(ONE, 1), - new Metric(TWO, 2), - new Metric(THREE, 3))) - .applyOutputNames(outputNamesById) - .build(); - - // Only original name - assertTrue(packet.metrics().containsKey(ONE_ID)); - - // Both names - assertTrue(packet.metrics().containsKey(TWO_ID)); - assertTrue(packet.metrics().containsKey(toMetricId("dos"))); - - // Only new name - assertFalse(packet.metrics().containsKey(THREE_ID)); - assertTrue(packet.metrics().containsKey(toMetricId("3"))); - - // Non-existent metric not added - assertFalse(packet.metrics().containsKey(NON_EXISTENT_ID)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java deleted file mode 100644 index 28912293fdb..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toMetricsPackets; -import static java.util.Collections.singleton; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - */ -public class JsonUtilTest { - @Test - public void json_model_gets_null_status_by_default() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.status_code); - assertNull(jsonModel.status_msg); - } - - @Test - public void status_is_included_in_json_model_when_explicitly_asked_for() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet), true).metrics.get(0); - assertNotNull(jsonModel.status_code); - assertNotNull(jsonModel.status_msg); - } - - @Test - public void timestamp_0_in_packet_is_translated_to_null_in_json_model() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .timestamp(0L) - .build(); - YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.timestamp); - } - - @Test - public void empty_consumers_is_translated_to_null_routing_in_json_model() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.routing); - } - - @Test - public void empty_json_string_yields_empty_packet_list() { - List<MetricsPacket.Builder> builders = toMetricsPackets(""); - assertTrue(builders.isEmpty()); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java deleted file mode 100644 index e91ff32e3b4..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Collections; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Tests for YamasJsonModel and YamasArrayJsonModel - * - * @author smorgrav - * @author gjoranv - */ -public class YamasJsonModelTest { - - private static final String EXPECTED_JSON = "{\"metrics\":[{\"status_code\":0,\"timestamp\":1400047900,\"application\":\"vespa.searchnode\",\"metrics\":{\"cpu\":55.5555555555555,\"memory_virt\":22222222222,\"memory_rss\":5555555555},\"dimensions\":{\"applicationName\":\"app\",\"tenantName\":\"tenant\",\"metrictype\":\"system\",\"instance\":\"searchnode\",\"applicationInstance\":\"default\",\"clustername\":\"cluster\"},\"routing\":{\"yamas\":{\"namespaces\":[\"Vespa\"]}},\"status_msg\":\"Data collected successfully\"}]}"; - - @Test - public void array_definition_creates_correct_json() throws IOException { - YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json"); - - YamasArrayJsonModel yamasData = new YamasArrayJsonModel(); - yamasData.add(jsonModel); - - assertEquals(EXPECTED_JSON, yamasData.serialize()); - } - - @Test - public void deserialize_serialize_roundtrip() throws IOException { - YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json"); - - // Do some sanity checking - assertEquals("vespa.searchnode", jsonModel.application); - assertEquals("Vespa", jsonModel.routing.get("yamas").namespaces.get(0)); - assertEquals(5.555555555E9, jsonModel.metrics.get("memory_rss"), 0.1d); //Not using custom double renderer - - // Serialize and verify - YamasArrayJsonModel yamasArray = new YamasArrayJsonModel(); - yamasArray.add(jsonModel); - String string = yamasArray.serialize(); - assertEquals(EXPECTED_JSON, string); - } - - @Test - public void deserialize_serialize_roundtrip_with_metrics_packet() throws IOException { - YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json"); - MetricsPacket metricsPacket = JsonUtil.toMetricsPacketBuilder(jsonModel).build(); - - // Do some sanity checking - assertEquals(toServiceId("vespa.searchnode"), metricsPacket.service); - assertEquals(toConsumerId("Vespa"), metricsPacket.consumers().get(0)); - assertEquals(5.555555555E9, metricsPacket.metrics().get(toMetricId("memory_rss")).doubleValue(), 0.1d); //Not using custom double rendrer - - // Serialize and verify - YamasArrayJsonModel yamasArray = JsonUtil.toYamasArray(Collections.singleton(metricsPacket), true); - String string = yamasArray.serialize(); - assertEquals(EXPECTED_JSON, string); - } - - @Test - public void missing_routing_object_makes_it_null() throws IOException { - // Read file that was taken from production (real -life example that is) - String filename = getClass().getClassLoader().getResource("yamas-array-no-routing.json").getFile(); - BufferedReader reader = Files.newBufferedReader(Paths.get(filename)); - ObjectMapper mapper = new ObjectMapper(); - YamasJsonModel jsonModel = mapper.readValue(reader, YamasJsonModel.class); - - // Do some sanity checking - assertNull(jsonModel.routing); - } - - private YamasJsonModel getYamasJsonModel(String testFile) throws IOException { - String filename = getClass().getClassLoader().getResource(testFile).getFile(); - BufferedReader reader = Files.newBufferedReader(Paths.get(filename)); - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(reader, YamasJsonModel.class); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java deleted file mode 100644 index 6bbf4ae1ef5..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.core.MonitoringConfig; -import ai.vespa.metricsproxy.core.VespaMetrics; -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.ServiceId; -import ai.vespa.metricsproxy.service.HttpMetricFetcher; -import ai.vespa.metricsproxy.service.MockHttpServer; -import ai.vespa.metricsproxy.service.VespaServices; -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import ai.vespa.metricsproxy.service.VespaServicesConfig.Service; - -import java.io.IOException; - -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static ai.vespa.metricsproxy.service.HttpMetricFetcher.STATE_PATH; - - -/** - * Setup and shutdown of config and servers for integration-style unit tests. - * - * @author hmusum - * @author gjoranv - */ -public class IntegrationTester implements AutoCloseable { - - static final String MONITORING_SYSTEM = "test-system"; - static final ConsumerId CUSTOM_CONSUMER_ID = toConsumerId("custom-consumer"); - static final String SERVICE_1_CONFIG_ID = "container/qrserver.0"; - static final String SERVICE_2_CONFIG_ID = "storage/cluster.storage/storage/0"; - - private final int httpPort; - private final int rpcPort; - private final RpcConnector connector; - private final MockHttpServer mockHttpServer; - private final VespaServices vespaServices; - - static { - HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests - } - - IntegrationTester(int httpPort, int rpcPort) { - if (httpPort == 0 || rpcPort == 0) { - throw new IllegalArgumentException("http port and rpc port must be defined"); - } - this.httpPort = httpPort; - this.rpcPort = rpcPort; - try { - mockHttpServer = new MockHttpServer(httpPort, null, STATE_PATH); - } catch (IOException e) { - throw new RuntimeException("Unable to start web server on port:" + httpPort); - } - - vespaServices = new VespaServices(servicesConfig(), monitoringConfig(), null); - MetricsConsumers consumers = new MetricsConsumers(consumersConfig()); - VespaMetrics vespaMetrics = new VespaMetrics(consumers, vespaServices); - ExternalMetrics externalMetrics = new ExternalMetrics(consumers); - ApplicationDimensions appDimensions = new ApplicationDimensions(applicationDimensionsConfig()); - NodeDimensions nodeDimensions = new NodeDimensions(nodeDimensionsConfig()); - - connector = new RpcConnector(rpcConnectorConfig()); - RpcServer server = new RpcServer(connector, vespaServices, new MetricsManager(vespaServices, vespaMetrics, externalMetrics, appDimensions, nodeDimensions)); - } - - MockHttpServer httpServer() { - return mockHttpServer; - } - - VespaServices vespaServices() { return vespaServices; } - - @Override - public void close() { - mockHttpServer.close(); - this.connector.stop(); - } - - private RpcConnectorConfig rpcConnectorConfig() { - return new RpcConnectorConfig.Builder() - .port(rpcPort) - .build(); - } - - private ConsumersConfig consumersConfig() { - return new ConsumersConfig.Builder() - .consumer(createConsumer(VESPA_CONSUMER_ID, "foo.count", "foo_count")) - .consumer(createConsumer(CUSTOM_CONSUMER_ID, "foo.count", "foo.count")) - .build(); - } - - private static Consumer.Builder createConsumer(ConsumerId consumerId, String metricName, String outputName) { - return new Consumer.Builder() - .name(consumerId.id) - .metric(new Consumer.Metric.Builder() - .dimension(new Consumer.Metric.Dimension.Builder().key("somekey").value("somevalue")) - .name(metricName) - .outputname(outputName)); - } - - private VespaServicesConfig servicesConfig() { - return new VespaServicesConfig.Builder() - .service(createService(toServiceId("qrserver"), SERVICE_1_CONFIG_ID, httpPort)) - .service(createService(toServiceId("storagenode"), SERVICE_2_CONFIG_ID, httpPort)) - .build(); - } - - private static Service.Builder createService(ServiceId serviceId, String configId, int port) { - return new Service.Builder() - .name(serviceId.id) - .configId(configId) - .port(port) - .healthport(port) - .dimension(new Service.Dimension.Builder().key("serviceDim").value("serviceDimValue")); - } - - private MonitoringConfig monitoringConfig() { - return new MonitoringConfig.Builder() - .systemName(MONITORING_SYSTEM) - .build(); - } - - private ApplicationDimensionsConfig applicationDimensionsConfig() { - return new ApplicationDimensionsConfig.Builder().build(); - } - - private NodeDimensionsConfig nodeDimensionsConfig() { - return new NodeDimensionsConfig.Builder().build(); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java deleted file mode 100644 index 20fb69e410e..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.service.MockHttpServer; -import ai.vespa.metricsproxy.service.VespaService; -import com.yahoo.jrt.Request; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.StringValue; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Target; -import com.yahoo.jrt.Transport; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author jobergum - * @author gjoranv - */ -public class RpcHealthMetricsTest { - - private static final String HEALTH_OK_RESPONSE = - TestUtil.getContents("health-check.response.json"); - private static final String HEALTH_FAILED_RESPONSE = - TestUtil.getContents("health-check-failed.response.json"); - private static final String WANTED_RPC_RESPONSE = - TestUtil.getContents("rpc-json-output-check.json").trim(); - - // see factory/doc/port-ranges.txt - private static final int httpPort = 18635; - private static final int rpcPort = 18636; - - @Test - public void expected_response_is_returned() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - - MockHttpServer mockHttpServer = tester.httpServer(); - mockHttpServer.setResponse(HEALTH_OK_RESPONSE); - List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - - assertThat(services.size(), is(1)); - VespaService qrserver = services.get(0); - HealthMetric h = qrserver.getHealth(); - assertNotNull("Health metric should never be null", h); - assertThat("Status failed, reason = " + h.getMessage(), h.isOk(), is(true)); - assertThat(h.getMessage(), is("WORKING")); - - mockHttpServer.setResponse(HEALTH_FAILED_RESPONSE); - h = qrserver.getHealth(); - assertNotNull("Health metric should never be null", h); - assertThat("Status should be failed" + h.getMessage(), h.isOk(), is(false)); - assertThat(h.getMessage(), is("SOMETHING FAILED")); - - String jsonRPCMessage = getHealthMetrics(qrserver.getMonitoringName()); - assertThat(jsonRPCMessage, is(WANTED_RPC_RESPONSE)); - } - } - - @Test - public void non_existent_service_name_returns_an_error_message() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - String jsonRPCMessage = getHealthMetrics("non-existing service"); - assertThat(jsonRPCMessage, is("105: No service with name 'non-existing service'")); - } - } - - private String getHealthMetrics(String service) { - Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); - Request req = new Request("getHealthMetricsForYamas"); - req.parameters().add(new StringValue(service)); - String returnValue; - - target.invokeSync(req, 20.0); - if (req.checkReturnTypes("s")) { - returnValue = req.returnValues().get(0).asString(); - } else { - System.out.println("RpcServer invocation failed " + req.errorCode() + ": " + req.errorMessage()); - returnValue = req.errorCode() + ": " + req.errorMessage(); - } - target.close(); - supervisor.transport().shutdown().join(); - - return returnValue; - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java deleted file mode 100644 index f264fd13ddc..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.service.VespaService; -import com.yahoo.jrt.Request; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.StringValue; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Target; -import com.yahoo.jrt.Transport; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static ai.vespa.metricsproxy.rpc.IntegrationTester.CUSTOM_CONSUMER_ID; -import static ai.vespa.metricsproxy.rpc.IntegrationTester.MONITORING_SYSTEM; -import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; -import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_2_CONFIG_ID; -import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author jobergum - * @author gjoranv - */ -public class RpcMetricsTest { - - private static final String METRICS_RESPONSE_CCL = - TestUtil.getContents("metrics-storage-simple.json").trim(); - - // see factory/doc/port-ranges.txt - private static final int httpPort = 18633; - private static final int rpcPort = 18634; - - @Test - public void testGetMetrics() throws Exception { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - tester.httpServer().setResponse(METRICS_RESPONSE_CCL); - List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - - assertThat("#Services should be 1 for config id " + SERVICE_1_CONFIG_ID, services.size(), is(1)); - - VespaService qrserver = services.get(0); - assertThat(qrserver.getMonitoringName(), is(MONITORING_SYSTEM + VespaService.SEPARATOR + "qrserver")); - - Metrics metrics = qrserver.getMetrics(); - assertThat("Fetched number of metrics is not correct", metrics.size(), is(2)); - Metric m = metrics.getMetric("foo.count"); - assertNotNull("Did not find expected metric with name 'foo.count'", m); - Metric m2 = metrics.getMetric("bar.count"); - assertNotNull("Did not find expected metric with name 'bar.count'", m2); - - // Setup RPC client - Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); - - verifyMetricsFromRpcRequest(qrserver, target); - - services = tester.vespaServices().getInstancesById(SERVICE_2_CONFIG_ID); - assertThat("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, services.size(), is(1)); - - VespaService storageService = services.get(0); - verfiyMetricsFromServiceObject(storageService); - - String metricsById = getMetricsById(storageService.getConfigId(), target); - assertThat(metricsById, is("'storage.cluster.storage.storage.0'.foo_count=1 ")); - - String jsonResponse = getMetricsForYamas("non-existing", target).trim(); - assertThat(jsonResponse, is("105: No service with name 'non-existing'")); - - verifyMetricsFromRpcRequestForAllServices(target); - - // Shutdown RPC - target.close(); - supervisor.transport().shutdown().join(); - } - } - - private static void verifyMetricsFromRpcRequest(VespaService service, Target target) throws JSONException { - String jsonResponse = getMetricsForYamas(service.getMonitoringName(), target).trim(); - JSONArray metrics = new JSONObject(jsonResponse).getJSONArray("metrics"); - assertThat("Expected 3 metric messages", metrics.length(), is(3)); - for (int i = 0; i < metrics.length() - 1; i++) { // The last "metric message" contains only status code/message - JSONObject jsonObject = metrics.getJSONObject(i); - assertFalse(jsonObject.has("status_code")); - assertFalse(jsonObject.has("status_msg")); - assertThat(jsonObject.getJSONObject("dimensions").getString("foo"), is("bar")); - assertThat(jsonObject.getJSONObject("dimensions").getString("bar"), is("foo")); - assertThat(jsonObject.getJSONObject("dimensions").getString("serviceDim"), is("serviceDimValue")); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").length(), is(1)); - if (jsonObject.getJSONObject("metrics").has("foo_count")) { - assertThat(jsonObject.getJSONObject("metrics").getInt("foo_count"), is(1)); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(VESPA_CONSUMER_ID.id)); - } else { - assertThat(jsonObject.getJSONObject("metrics").getInt("foo.count"), is(1)); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(CUSTOM_CONSUMER_ID.id)); - } - } - - verifyStatusMessage(metrics.getJSONObject(metrics.length() - 1)); - } - - private void verfiyMetricsFromServiceObject(VespaService service) { - Metrics storageMetrics = service.getMetrics(); - assertThat(storageMetrics.size(), is(2)); - Metric foo = storageMetrics.getMetric("foo.count"); - assertNotNull("Did not find expected metric with name 'foo.count'", foo); - assertThat("Expected 2 dimensions for metric foo", foo.getDimensions().size(), is(2)); - assertThat("Metric foo did not contain correct dimension mapping for key = foo.count", foo.getDimensions().containsKey(toDimensionId("foo")), is(true)); - assertThat("Metric foo did not contain correct dimension", foo.getDimensions().get(toDimensionId("foo")), is("bar")); - assertThat("Metric foo did not contain correct dimension", foo.getDimensions().containsKey(toDimensionId("bar")), is(true)); - assertThat("Metric foo did not contain correct dimension for key = bar", foo.getDimensions().get(toDimensionId("bar")), is("foo")); - } - - private void verifyMetricsFromRpcRequestForAllServices(Target target) throws JSONException { - // Verify that metrics for all services can be retrieved in one request. - String allServicesResponse = getMetricsForYamas(ALL_SERVICES, target).trim(); - JSONArray allServicesMetrics = new JSONObject(allServicesResponse).getJSONArray("metrics"); - assertThat(allServicesMetrics.length(), is(5)); - } - - @Test - public void testGetAllMetricNames() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - - tester.httpServer().setResponse(METRICS_RESPONSE_CCL); - List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - - assertThat(services.size(), is(1)); - Metrics metrics = services.get(0).getMetrics(); - assertThat("Fetched number of metrics is not correct", metrics.size(), is(2)); - Metric m = metrics.getMetric("foo.count"); - assertNotNull("Did not find expected metric with name 'foo.count'", m); - - - Metric m2 = metrics.getMetric("bar.count"); - assertNotNull("Did not find expected metric with name 'bar'", m2); - - // Setup RPC - Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); - - String response = getAllMetricNamesForService(services.get(0).getMonitoringName(), VESPA_CONSUMER_ID, target); - assertThat(response, is("foo.count=ON;output-name=foo_count,bar.count=OFF,")); - - // Shutdown RPC - target.close(); - supervisor.transport().shutdown().join(); - } - } - - private static String getMetricsForYamas(String service, Target target) { - Request req = new Request("getMetricsForYamas"); - req.parameters().add(new StringValue(service)); - return invoke(req, target); - } - - private String getMetricsById(String service, Target target) { - Request req = new Request("getMetricsById"); - req.parameters().add(new StringValue(service)); - return invoke(req, target); - } - - private String getAllMetricNamesForService(String service, ConsumerId consumer, Target target) { - Request req = new Request("getAllMetricNamesForService"); - req.parameters().add(new StringValue(service)); - req.parameters().add(new StringValue(consumer.id)); - return invoke(req, target); - } - - private static String invoke(Request req, Target target) { - String returnValue; - target.invokeSync(req, 20.0); - if (req.checkReturnTypes("s")) { - returnValue = req.returnValues().get(0).asString(); - } else { - System.out.println(req.methodName() + " from rpcserver - Invocation failed " - + req.errorCode() + ": " + req.errorMessage()); - returnValue = req.errorCode() + ": " + req.errorMessage(); - } - return returnValue; - } - - private static void verifyStatusMessage(JSONObject jsonObject) throws JSONException { - assertThat(jsonObject.getInt("status_code"), is(0)); - assertThat(jsonObject.getString("status_msg"), notNullValue()); - assertThat(jsonObject.getString("application"), notNullValue()); - assertThat(jsonObject.getString("routing"), notNullValue()); - assertThat(jsonObject.length(), is(4)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java deleted file mode 100644 index bd61b8443aa..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknown - */ -public class ConfigSentinelClientTest { - - @Test - public void testConfigSentinelClient() { - ConfigSentinelDummy configsentinel = new ConfigSentinelDummy(); - List<VespaService> services = new ArrayList<>(); - VespaService docproc = new VespaService("docprocservice", "docproc/cluster.x.indexing/0"); - VespaService searchnode4 = new VespaService("searchnode4", "search/cluster.x/g0/c1/r1"); - VespaService qrserver = new VespaService("qrserver", "container/qrserver.0"); - - services.add(searchnode4); - services.add(qrserver); - services.add(docproc); - - MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel); - client.updateServiceStatuses(services); - - assertThat(qrserver.getPid(), is(6520)); - assertThat(qrserver.getState(), is("RUNNING")); - assertThat(qrserver.isAlive(), is(true)); - assertThat(searchnode4.getPid(), is(6534)); - assertThat(searchnode4.getState(), is("RUNNING")); - assertThat(searchnode4.isAlive(), is(true)); - - assertThat(docproc.getPid(), is(-1)); - assertThat(docproc.getState(), is("FINISHED")); - assertThat(docproc.isAlive(), is(false)); - - - configsentinel.reConfigure(); - - client.ping(docproc); - assertThat(docproc.getPid(), is(100)); - assertThat(docproc.getState(), is("RUNNING")); - assertThat(docproc.isAlive(), is(true)); - - //qrserver has yet not been checked - assertThat(qrserver.isAlive(), is(true)); - - client.updateServiceStatuses(services); - - assertThat(docproc.getPid(), is(100)); - assertThat(docproc.getState(), is("RUNNING")); - assertThat(docproc.isAlive(), is(true)); - //qrserver is no longer running on this node - so should be false - assertThat(qrserver.isAlive(), is(false)); - } - - @Test - public void testElastic() throws Exception { - String response = "container state=RUNNING mode=AUTO pid=14338 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"get/container.0\"\n" + - "container-clustercontroller state=RUNNING mode=AUTO pid=25020 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/cluster-controllers/0\"\n" + - "distributor state=RUNNING mode=AUTO pid=25024 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/distributor/0\"\n" + - "docprocservice state=RUNNING mode=AUTO pid=11973 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.search.indexing/0\"\n" + - "logd state=RUNNING mode=AUTO pid=25016 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/vespa19.dev.gq1.yahoo.com/logd\"\n" + - "logserver state=RUNNING mode=AUTO pid=25018 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" + - "metricsproxy state=RUNNING mode=AUTO pid=13107 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/vespa19.dev.gq1.yahoo.com/metricsproxy\"\n" + - "searchnode state=RUNNING mode=AUTO pid=25023 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/0\"\n" + - "slobrok state=RUNNING mode=AUTO pid=25019 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" + - "topleveldispatch state=RUNNING mode=AUTO pid=25026 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/tlds/tld.0\"\n" + - "\n"; - - ConfigSentinelDummy configsentinel = new ConfigSentinelDummy(response); - List<VespaService> services = new ArrayList<>(); - - VespaService container = VespaService.create("container", "get/container.0", -1); - - VespaService containerClusterController = - VespaService.create("container-clustercontroller", "get/container.0", -1); - - VespaService notPresent = VespaService.create("dummy","fake", -1); - - services.add(container); - services.add(containerClusterController); - services.add(notPresent); - - MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel); - client.updateServiceStatuses(services); - assertThat(container.isAlive(),is(true)); - assertThat(container.getPid(),is(14338)); - assertThat(container.getState(),is("RUNNING")); - - assertThat(containerClusterController.isAlive(),is(true)); - assertThat(containerClusterController.getPid(),is(25020)); - assertThat(containerClusterController.getState(),is("RUNNING")); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java deleted file mode 100644 index 108f5c18e1d..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ -package ai.vespa.metricsproxy.service; - -/** - * @author Eirik Nygaard - */ -public class ConfigSentinelDummy { - private String serviceList = - "docprocservice state=FINISHED mode=MANUAL pid=6555 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.x.indexing/0\"\n" - + "distributor state=RUNNING mode=AUTO pid=6548 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/distributor/0\"\n" - + "fleetcontroller state=RUNNING mode=AUTO pid=6543 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/fleetcontroller/0\"\n" - + "storagenode state=RUNNING mode=AUTO pid=6539 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/storage/0\"\n" - + "searchnode4 state=RUNNING mode=AUTO pid=6534 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r1\"\n" - + "qrserver2 state=RUNNING mode=AUTO pid=6521 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.1\"\n" - + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" - + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n" - + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n" - + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n" - + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n" - + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n" - + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n" - + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" - + "searchnode3 state=RUNNING mode=AUTO pid=6529 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r0\"\n" - + "searchnode state=RUNNING mode=AUTO pid=6526 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r0\"\n" - + "qrserver state=RUNNING mode=AUTO pid=6520 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.0\"\n" - + "\n"; - - - public ConfigSentinelDummy() { - } - - public ConfigSentinelDummy(String response) { - serviceList = response; - } - - public void reConfigure() { - this.serviceList = "docprocservice state=RUNNING mode=AUTO pid=100 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.x.indexing/0\"\n" - + "distributor state=RUNNING mode=AUTO pid=6548 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/distributor/0\"\n" - + "fleetcontroller state=RUNNING mode=AUTO pid=6543 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/fleetcontroller/0\"\n" - + "storagenode state=RUNNING mode=AUTO pid=6539 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/storage/0\"\n" - + "searchnode4 state=RUNNING mode=AUTO pid=6534 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r1\"\n" - + "qrserver2 state=RUNNING mode=AUTO pid=6521 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.1\"\n" - + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" - + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n" - + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n" - + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n" - + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n" - + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n" - + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n" - + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" - + "searchnode3 state=RUNNING mode=AUTO pid=6529 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r0\"\n" - + "searchnode state=RUNNING mode=AUTO pid=6526 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r0\"\n" - + "\n"; - } - - public String getServiceList() { - return serviceList; - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java deleted file mode 100644 index 4174b18f3a7..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.Metric; -import org.json.JSONException; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknown - */ -public class ContainerServiceTest { - - private MockHttpServer service; - private int csPort; - - @BeforeClass - public static void init() { - HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests - } - - @Before - public void setupHTTPServer() { - csPort = 18637; // see factory/doc/port-ranges.txt - try { - String response = TestUtil.getContents("metrics-container-state-multi-chain.json"); - service = new MockHttpServer(csPort, response, HttpMetricFetcher.METRICS_PATH); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void testMultipleQueryDimensions() throws JSONException { - int count = 0; - VespaService service = VespaService.create("service1", "id", csPort); - for (Metric m : service.getMetrics().getMetrics()) { - if (m.getName().equals("queries.rate")) { - count++; - System.out.println("Name: " + m.getName() + " value: " + m.getValue()); - if (m.getDimensions().get(toDimensionId("chain")).equals("asvBlendingResult")) { - assertThat((Double)m.getValue(), is(26.4)); - } else if (m.getDimensions().get(toDimensionId("chain")).equals("blendingResult")) { - assertThat((Double)m.getValue(), is(0.36666666666666664)); - } else { - assertThat("Unknown unknown chain", false, is(true)); - } - } - } - assertThat(count, is(2)); - } - - @After - public void shutdown() { - this.service.close(); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java deleted file mode 100644 index 380a992aead..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; - -/** - * @author Unknown - */ -public class DummyService extends VespaService { - static final String NAME = "dummy"; - public static final String METRIC_1 = "c.test"; - public static final String METRIC_2 = "val"; - - private final int num; - - public DummyService(int num, String configid) { - super(NAME, NAME + num, configid); - this.num = num; - } - - @Override - public Metrics getMetrics() { - Metrics m = new Metrics(); - - long timestamp = System.currentTimeMillis() / 1000; - m.add(new Metric(METRIC_1, 5 * num + 1, timestamp)); - m.add(new Metric(METRIC_2, 1.3 * num + 1.05, timestamp)); - - return m; - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java deleted file mode 100644 index 27e1bb97943..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.Metrics; -import org.junit.Test; - -import java.io.File; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknowm - */ -public class MetricsFetcherTest { - private static int port = 9; //port number is not used in this test - - @Test - public void testStateFormatMetricsParse() { - String jsonData = TestUtil.getContents("metrics-state.json"); - RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); - Metrics metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(10)); - assertThat("Wrong value for metric", metrics.get("query_hits.count").intValue(), is(28)); - assertThat("Wrong value for metric ", metrics.get("queries.rate").doubleValue(), is(0.4667)); - assertThat("Wrong timestamp", metrics.getTimeStamp(), is(1334134700L)); - } - - @Test - public void testEmptyJson() { - String jsonData = "{}"; - RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); - Metrics metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(0)); - } - - @Test - public void testErrors() { - String jsonData; - Metrics metrics; - - RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); - - jsonData = ""; - metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(0)); - - jsonData = "{\n" + - "\"status\" : {\n" + - " \"code\" : \"up\",\n" + - " \"message\" : \"Everything ok here\"\n" + - "}\n" + - "}"; - metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(0)); - - jsonData = "{\n" + - "\"status\" : {\n" + - " \"code\" : \"up\",\n" + - " \"message\" : \"Everything ok here\"\n" + - "},\n" + - "\"metrics\" : {\n" + - " \"snapshot\" : {\n" + - " \"from\" : 1334134640.089,\n" + - " \"to\" : 1334134700.088\n" + - " },\n" + - " \"values\" : [\n" + - " {\n" + - " \"name\" : \"queries\",\n" + - " \"description\" : \"Number of queries executed during snapshot interval\",\n" + - " \"values\" : {\n" + - " \"count\" : null,\n" + - " \"rate\" : 0.4667\n" + - " },\n" + - " \"dimensions\" : {\n" + - " \"searcherid\" : \"x\"\n" + - " }\n" + - " }\n" + "" + - " ]\n" + - "}\n" + - "}"; - - metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(0)); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java deleted file mode 100644 index 917c529e63e..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ -package ai.vespa.metricsproxy.service; - -import com.yahoo.log.LogLevel; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - -/** - * Mock config sentinel - * - * @author hmusum - */ -public class MockConfigSentinelClient extends ConfigSentinelClient { - private final ConfigSentinelDummy configSentinel; - private final static Logger log = Logger.getLogger(MockConfigSentinelClient.class.getPackage().getName()); - - public MockConfigSentinelClient(ConfigSentinelDummy configSentinel) { - super(); - this.configSentinel = configSentinel; - } - - @Override - protected synchronized void setStatus(List<VespaService> services) throws Exception { - List<VespaService> updatedServices = new ArrayList<>(); - String[] lines = configSentinel.getServiceList().split("\n"); - for (String line : lines) { - if (line.equals("")) { - break; - } - - VespaService s = parseServiceString(line, services); - if (s != null) { - updatedServices.add(s); - } - } - - //Check if there are services that were not found in - //from the sentinel - for (VespaService s : services) { - if (!updatedServices.contains(s)) { - log.log(LogLevel.DEBUG, "Service " + s + " is no longer found with sentinel - setting alive = false"); - s.setAlive(false); - } - } - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java deleted file mode 100644 index fdf2fae3081..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; -import com.sun.net.httpserver.HttpServer; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetSocketAddress; - -/** - * @author jobergum - */ -public class MockHttpServer { - - private String response; - private HttpServer server; - - /** - * Mock http server that will return response as body - * - * @param port the port to listen to - * @param response the response to return along with 200 OK - * @param path the file path that the server will accept requests for. E.g /state/v1/metrics - */ - public MockHttpServer(int port, String response, String path) throws IOException { - this.response = response; - this.server = HttpServer.create(new InetSocketAddress(port), 10); - this.server.createContext(path, new MyHandler()); - this.server.setExecutor(null); // creates a default executor - this.server.start(); - System.out.println("Started web server on port " + port); - } - - public synchronized void setResponse(String r) { - this.response = r; - } - - public void close() { - this.server.stop(0); - } - - private class MyHandler implements HttpHandler { - public void handle(HttpExchange t) throws IOException { - t.sendResponseHeaders(200, response.length()); - OutputStream os = t.getResponseBody(); - os.write(response.getBytes()); - os.close(); - } - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java deleted file mode 100644 index a42d52b7ea6..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknown - */ -public class SystemPollerTest { - - @Test - public void testSystemPoller() { - DummyService s = new DummyService(0, "id"); - List<VespaService> services = new ArrayList<>(); - services.add(s); - - SystemPoller poller = new SystemPoller(services, 60*5); - assertThat(s.isAlive(), is(false)); - services.remove(0); - poller.setServices(services); - long n = poller.getPidJiffies(s); - assertThat(n, is(0L)); - long[] memusage = poller.getMemoryUsage(s); - assertThat(memusage[0], is(0L)); - assertThat(memusage[1], is(0L)); - } - - @Test - public void testCPUJiffies() { - String line = "cpu1 102180864 789 56766899 12800020140 1654757 0 0"; - CpuJiffies n = new CpuJiffies(line); - assertThat(n.getCpuId(), is(1)); - assertThat(n.getTotalJiffies(), is(12960623449L)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java deleted file mode 100644 index 13be98db23a..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.Metrics; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknown - */ -public class VespaServiceTest { - private MockHttpServer service; - private int csPort; - private static final String response; - - static { - response = TestUtil.getContents("metrics-state.json"); - HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests - } - - @Before - public void setupHTTPServer() { - csPort = 18632; // see factory/doc/port-ranges.txt - try { - service = new MockHttpServer(csPort, response, HttpMetricFetcher.METRICS_PATH); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void testService() { - VespaService service = new VespaService("qrserver", "container/qrserver.0"); - assertThat(service.getServiceName(), is("qrserver")); - assertThat(service.getInstanceName(), is("qrserver")); - assertThat(service.getPid(), is(-1)); - assertThat(service.getConfigId(), is("container/qrserver.0")); - - - service = VespaService.create("qrserver2", "container/qrserver.0", -1); - assertThat(service.getServiceName(), is("qrserver")); - assertThat(service.getInstanceName(), is("qrserver2")); - assertThat(service.getPid(), is(-1)); - assertThat(service.getConfigId(), is("container/qrserver.0")); - } - - @Test - // TODO: Make it possible to test this without running a HTTP server to create the response - public void testMetricsFetching() { - VespaService service = VespaService.create("service1", "id", csPort); - Metrics metrics = service.getMetrics(); - assertThat(metrics.getMetric("queries.count").getValue().intValue(), is(28)); - - // Shutdown server and check that no metrics are returned (should use empty metrics - // when unable to fetch new metrics) - shutdown(); - - metrics = service.getMetrics(); - assertThat(metrics.size(), is(0)); - } - - @After - public void shutdown() { - this.service.close(); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java deleted file mode 100644 index bd0b670ca35..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import com.google.common.collect.ImmutableList; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * TODO: add more tests - * - * @author gjoranv - */ -public class VespaServicesTest { - - @Test - public void services_can_be_retrieved_from_monitoring_name() { - List<VespaService> dummyServices = ImmutableList.of( - new DummyService(0, "dummy/id/0"), - new DummyService(1, "dummy/id/1")); - VespaServices services = new VespaServices(dummyServices); - - assertThat(services.getMonitoringServices("vespa.dummy").size(), is(2)); - } - - @Test - public void all_services_can_be_retrieved_by_using_special_name() { - List<VespaService> dummyServices = ImmutableList.of( - new DummyService(0, "dummy/id/0")); - VespaServices services = new VespaServices(dummyServices); - - assertThat(services.getMonitoringServices(ALL_SERVICES).size(), is(1)); - } - -} diff --git a/metrics-proxy/src/test/resources/health-check-failed.response.json b/metrics-proxy/src/test/resources/health-check-failed.response.json deleted file mode 100644 index e118f10ec5e..00000000000 --- a/metrics-proxy/src/test/resources/health-check-failed.response.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "status": { - "code": "down", - "message":"SOMETHING FAILED" - }, - "metrics": [] - } diff --git a/metrics-proxy/src/test/resources/health-check.response.json b/metrics-proxy/src/test/resources/health-check.response.json deleted file mode 100644 index 8e3858ec5d8..00000000000 --- a/metrics-proxy/src/test/resources/health-check.response.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "status": { - "code": "OK", - "message":"WORKING" - } - }
\ No newline at end of file diff --git a/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json b/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json deleted file mode 100644 index 76d0be50cca..00000000000 --- a/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json +++ /dev/null @@ -1,281 +0,0 @@ -{ - "metrics": { - "snapshot": { - "from": 1.383132197389E9, - "to": 1.383132257389E9 - }, - "values": [ - { - "name": "search_connections", - "values": { - "average": 3.459204315576534, - "count": 1, - "last": 3.459204315576534, - "max": 3.459204315576534, - "min": 3.459204315576534, - "rate": 0.016666666666666666 - } - }, - { - "name": "active_queries", - "values": { - "average": 0, - "count": 1, - "last": 0, - "max": 0, - "min": 0, - "rate": 0.016666666666666666 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumFailedResponseWrites", - "values": { - "count": 85, - "rate": 1.4166666666666667 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumFailedResponses", - "values": { - "count": 8, - "rate": 0.13333333333333333 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumConnections", - "values": { - "count": 1630, - "rate": 27.166666666666668 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumSuccessfulResponses", - "values": { - "count": 1621, - "rate": 27.016666666666666 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNetworkLatency", - "values": { - "average": 0.11715958713775308, - "count": 20152, - "last": 0, - "max": 55, - "min": 0, - "rate": 335.8666666666667 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumSuccessfulResponseWrites", - "values": { - "count": 20152, - "rate": 335.8666666666667 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverTotalSuccessfulResponseLatency", - "values": { - "average": 90.88401253918495, - "count": 1595, - "last": 80, - "max": 233, - "min": 0, - "rate": 26.583333333333332 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumRequests", - "values": { - "count": 1633, - "rate": 27.216666666666665 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverTotalFailedResponseLatency", - "values": { - "average": 0.75, - "count": 8, - "last": 1, - "max": 1, - "min": 0, - "rate": 0.13333333333333333 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "query_latency", - "values": { - "average": 83.35949367088608, - "count": 1580, - "last": 61, - "max": 224, - "min": 12, - "rate": 26.333333333333332 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "max_query_latency", - "values": { - "average": 83.35949367088608, - "count": 1580, - "last": 61, - "max": 224, - "min": 12, - "rate": 26.333333333333332 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "peak_qps", - "values": { - "average": 25.87656434951563, - "count": 6, - "last": 23.681592039800993, - "max": 29.7659845295212, - "min": 23.681592039800993, - "rate": 0.1 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "queries", - "values": { - "count": 1584, - "rate": 26.4 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "mean_query_latency", - "values": { - "average": 83.35949367088608, - "count": 1580, - "last": 61, - "max": 224, - "min": 12, - "rate": 26.333333333333332 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "hits_per_query", - "values": { - "average": 173.70126582278482, - "count": 1580, - "last": 175, - "max": 175, - "min": 5, - "rate": 26.333333333333332 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "query_latency", - "values": { - "average": 39.40909090909091, - "count": 22, - "last": 26, - "max": 174, - "min": 13, - "rate": 0.36666666666666664 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "max_query_latency", - "values": { - "average": 39.40909090909091, - "count": 22, - "last": 26, - "max": 174, - "min": 13, - "rate": 0.36666666666666664 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "peak_qps", - "values": { - "average": 0.5890415170417276, - "count": 3, - "last": 0.40488561981240295, - "max": 0.864528399757932, - "min": 0.40488561981240295, - "rate": 0.05 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "queries", - "values": { - "count": 22, - "rate": 0.36666666666666664 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "mean_query_latency", - "values": { - "average": 39.40909090909091, - "count": 22, - "last": 26, - "max": 174, - "min": 13, - "rate": 0.36666666666666664 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "hits_per_query", - "values": { - "average": 47.5, - "count": 22, - "last": 28, - "max": 176, - "min": 5, - "rate": 0.36666666666666664 - } - } - ] - }, - "status": {"code": "up"}, - "time": 1383132269767 -}
\ No newline at end of file diff --git a/metrics-proxy/src/test/resources/metrics-state.json b/metrics-proxy/src/test/resources/metrics-state.json deleted file mode 100644 index b7773e5fb8b..00000000000 --- a/metrics-proxy/src/test/resources/metrics-state.json +++ /dev/null @@ -1,42 +0,0 @@ -{ -"status" : { - "code" : "up", - "message" : "Everything ok here" -}, -"metrics" : { - "snapshot" : { - "from" : 1334134640.089, - "to" : 1334134700.088 - }, - "values" : [ - { - "name" : "queries", - "description" : "Number of queries executed during snapshot interval", - "values" : { - "count" : 28, - "rate" : 0.4667 - }, - "dimensions" : { - "searcherid" : "x" - } - }, - { - "name" : "query_hits", - "description" : "Number of documents matched per query during snapshot interval", - "values" : { - "count" : 28, - "rate" : 0.4667, - "average" : 128.3, - "min" : 0, - "max" : 10000, - "sum" : 3584, - "median" : 124.0, - "std_deviation": 5.43 - }, - "dimensions" : { - "searcherid" : "x" - } - } - ] -} -} diff --git a/metrics-proxy/src/test/resources/metrics-storage-simple.json b/metrics-proxy/src/test/resources/metrics-storage-simple.json deleted file mode 100644 index 00715b52046..00000000000 --- a/metrics-proxy/src/test/resources/metrics-storage-simple.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "status" : { - "code" : "up", - "message": "All good" - }, - - "metrics" : { - "snapshot" : { - "from": 1335523285, - "to": 1335525685 - }, - "values": [ - - { - "name" : "foo", - "values" : { - "count" : 1 - }, - "dimensions" : { - "foo": "bar", - "bar" : "foo" - } - }, - - { - "name" : "bar", - "values" : { - "count" : 2 - }, - "dimensions" : { - "d0": "d0val", - "d1" : "d1val" - } - } - ] - - } -} diff --git a/metrics-proxy/src/test/resources/rpc-json-output-check.json b/metrics-proxy/src/test/resources/rpc-json-output-check.json deleted file mode 100644 index 701a06d82b2..00000000000 --- a/metrics-proxy/src/test/resources/rpc-json-output-check.json +++ /dev/null @@ -1 +0,0 @@ -{"metrics":[{"status_code":1,"application":"test-system.qrserver","dimensions":{"metrictype":"health","instance":"qrserver"},"status_msg":"SOMETHING FAILED"}]}
\ No newline at end of file diff --git a/metrics-proxy/src/test/resources/yamas-array-no-routing.json b/metrics-proxy/src/test/resources/yamas-array-no-routing.json deleted file mode 100644 index 8f21e8253b9..00000000000 --- a/metrics-proxy/src/test/resources/yamas-array-no-routing.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "status_code" : 0, - "timestamp" : 1400047900, - "application" : "vespa.searchnode", - "metrics" : { - "cpu" : 55.5555555555555, - "memory_virt" : 22222222222, - "memory_rss" : 5555555555 - }, - "dimensions" : { - "applicationName" : "app", - "tenantName" : "tenant", - "metrictype" : "system", - "instance" : "searchnode", - "applicationInstance" : "default", - "clustername" : "cluster" - }, - "status_msg" : "Data collected successfully" -} diff --git a/metrics-proxy/src/test/resources/yamas-array.json b/metrics-proxy/src/test/resources/yamas-array.json deleted file mode 100644 index c9293623b25..00000000000 --- a/metrics-proxy/src/test/resources/yamas-array.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "status_code" : 0, - "timestamp" : 1400047900, - "application" : "vespa.searchnode", - "metrics" : { - "cpu" : 55.5555555555555, - "memory_virt" : 22222222222, - "memory_rss" : 5555555555 - }, - "dimensions" : { - "applicationName" : "app", - "tenantName" : "tenant", - "metrictype" : "system", - "instance" : "searchnode", - "applicationInstance" : "default", - "clustername" : "cluster" - }, - "routing" : { - "yamas" : { - "namespaces" : [ - "Vespa" - ] - } - }, - "status_msg" : "Data collected successfully" -} diff --git a/metrics/src/tests/CMakeLists.txt b/metrics/src/tests/CMakeLists.txt index 2c69b83d4b7..11d59e63c30 100644 --- a/metrics/src/tests/CMakeLists.txt +++ b/metrics/src/tests/CMakeLists.txt @@ -5,6 +5,8 @@ vespa_add_executable(metrics_gtest_runner_app TEST SOURCES countmetrictest.cpp + loadmetrictest.cpp + summetrictest.cpp gtest_runner.cpp DEPENDS metrics @@ -23,9 +25,7 @@ vespa_add_executable(metrics_testrunner_app TEST testrunner.cpp valuemetrictest.cpp metricsettest.cpp - summetrictest.cpp metricmanagertest.cpp - loadmetrictest.cpp snapshottest.cpp stresstest.cpp metrictest.cpp diff --git a/metrics/src/tests/loadmetrictest.cpp b/metrics/src/tests/loadmetrictest.cpp index 5cbec2c4fae..b89628f4476 100644 --- a/metrics/src/tests/loadmetrictest.cpp +++ b/metrics/src/tests/loadmetrictest.cpp @@ -3,7 +3,7 @@ #include <vespa/metrics/valuemetric.h> #include <vespa/metrics/loadmetric.hpp> #include <vespa/metrics/summetric.hpp> -#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/gtest/gtest.h> namespace metrics { @@ -20,30 +20,11 @@ struct LoadTypeSetImpl : public LoadTypeSet { const LoadType& lt(LoadTypeSet::operator[](i)); if (lt.getName() == name) return lt; } - CPPUNIT_FAIL("No load type with name " + name); - return operator[](0); // Should never get here + abort(); } }; -struct LoadMetricTest : public CppUnit::TestFixture { - void testNormalUsage(); - void testClone(Metric::CopyType); - void testInactiveCopy() { testClone(Metric::INACTIVE); } - void testActiveCopy() { testClone(Metric::CLONE); } - void testAdding(); - - CPPUNIT_TEST_SUITE(LoadMetricTest); - CPPUNIT_TEST(testNormalUsage); - CPPUNIT_TEST(testActiveCopy); - CPPUNIT_TEST(testInactiveCopy); - CPPUNIT_TEST(testAdding); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(LoadMetricTest); - -void -LoadMetricTest::testNormalUsage() +TEST(LoadMetricTest, test_normal_usage) { LoadTypeSetImpl loadTypes; loadTypes.add(32, "foo").add(1000, "bar"); @@ -76,7 +57,7 @@ namespace { } void -LoadMetricTest::testClone(Metric::CopyType copyType) +test_clone(Metric::CopyType copyType) { LoadTypeSetImpl loadTypes; loadTypes.add(32, "foo").add(1000, "bar"); @@ -87,7 +68,7 @@ LoadMetricTest::testClone(Metric::CopyType copyType) std::vector<Metric::UP> ownerList; MetricSet::UP copy(dynamic_cast<MetricSet*>(top.clone(ownerList, copyType, 0, true))); - CPPUNIT_ASSERT(copy.get()); + ASSERT_TRUE(copy.get()); std::string expected = "top:\n" @@ -101,12 +82,21 @@ LoadMetricTest::testClone(Metric::CopyType copyType) " bar:\n" " tack average=0 last=0 count=0 total=0"; - CPPUNIT_ASSERT_EQUAL(expected, std::string(top.toString(true))); - CPPUNIT_ASSERT_EQUAL(expected, std::string(copy->toString(true))); + EXPECT_EQ(expected, std::string(top.toString(true))); + EXPECT_EQ(expected, std::string(copy->toString(true))); } -void -LoadMetricTest::testAdding() +TEST(LoadMetricTest, test_inactive_copy) +{ + test_clone(Metric::INACTIVE); +} + +TEST(LoadMetricTest, test_active_copy) +{ + test_clone(Metric::CLONE); +} + +TEST(LoadMetricTest, test_adding) { LoadTypeSetImpl loadTypes; loadTypes.add(32, "foo").add(1000, "bar"); @@ -118,7 +108,7 @@ LoadMetricTest::testAdding() std::vector<Metric::UP> ownerList; MetricSet::UP copy(dynamic_cast<MetricSet*>( top.clone(ownerList, Metric::INACTIVE, 0, false))); - CPPUNIT_ASSERT(copy.get()); + ASSERT_TRUE(copy.get()); top.reset(); @@ -132,8 +122,7 @@ LoadMetricTest::testAdding() " foo:\n" " tack average=5 last=5 min=5 max=5 count=1 total=5"; - CPPUNIT_ASSERT_EQUAL(expected, std::string(copy->toString(true))); - + EXPECT_EQ(expected, std::string(copy->toString(true))); } -} // documentapi +} diff --git a/metrics/src/tests/summetrictest.cpp b/metrics/src/tests/summetrictest.cpp index 32c5ee2c309..e3d58659daf 100644 --- a/metrics/src/tests/summetrictest.cpp +++ b/metrics/src/tests/summetrictest.cpp @@ -1,29 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/metrics/metrics.h> -#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/gtest/gtest.h> namespace metrics { -struct SumMetricTest : public CppUnit::TestFixture { - void testLongCountMetric(); - void testAverageMetric(); - void testMetricSet(); - void testRemove(); - void testStartValue(); - - CPPUNIT_TEST_SUITE(SumMetricTest); - CPPUNIT_TEST(testLongCountMetric); - CPPUNIT_TEST(testAverageMetric); - CPPUNIT_TEST(testMetricSet); - CPPUNIT_TEST(testRemove); - CPPUNIT_TEST(testStartValue); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(SumMetricTest); - -void -SumMetricTest::testLongCountMetric() +TEST(SumMetricTest, test_long_count_metric) { MetricSet parent("parent", {}, ""); SumMetric<LongCountMetric> sum("foo", {}, "foodesc", &parent); @@ -40,12 +21,12 @@ SumMetricTest::testLongCountMetric() // Verify XML output. Should be in register order. std::string expected("foo count=10"); - CPPUNIT_ASSERT_EQUAL(expected, sum.toString()); - CPPUNIT_ASSERT_EQUAL(int64_t(10), sum.getLongValue("value")); + EXPECT_EQ(expected, sum.toString()); + EXPECT_EQ(int64_t(10), sum.getLongValue("value")); } -void -SumMetricTest::testAverageMetric() { +TEST(SumMetricTest, test_average_metric) +{ MetricSet parent("parent", {}, ""); SumMetric<LongAverageMetric> sum("foo", {}, "foodesc", &parent); @@ -61,14 +42,14 @@ SumMetricTest::testAverageMetric() { // Verify XML output. Should be in register order. std::string expected("foo average=5 last=7 min=3 max=7 count=2 total=10"); - CPPUNIT_ASSERT_EQUAL(expected, sum.toString()); - CPPUNIT_ASSERT_EQUAL(int64_t(5), sum.getLongValue("value")); - CPPUNIT_ASSERT_EQUAL(int64_t(3), sum.getLongValue("min")); - CPPUNIT_ASSERT_EQUAL(int64_t(7), sum.getLongValue("max")); + EXPECT_EQ(expected, sum.toString()); + EXPECT_EQ(int64_t(5), sum.getLongValue("value")); + EXPECT_EQ(int64_t(3), sum.getLongValue("min")); + EXPECT_EQ(int64_t(7), sum.getLongValue("max")); } -void -SumMetricTest::testMetricSet() { +TEST(SumMetricTest, test_metric_set) +{ MetricSet parent("parent", {}, ""); SumMetric<MetricSet> sum("foo", {}, "bar", &parent); @@ -82,23 +63,22 @@ SumMetricTest::testMetricSet() { sum.addMetricToSum(set1); sum.addMetricToSum(set2); - // Give them some values + // Give them some values v1.addValue(3); v2.addValue(7); v3.inc(2); v4.inc(); - // Verify XML output. Should be in register order. + // Verify XML output. Should be in register order. std::string expected("'\n" "foo:\n" " c average=3 last=3 min=3 max=3 count=1 total=3\n" " e count=2'" ); - CPPUNIT_ASSERT_EQUAL(expected, "'\n" + sum.toString() + "'"); + EXPECT_EQ(expected, "'\n" + sum.toString() + "'"); } -void -SumMetricTest::testRemove() +TEST(SumMetricTest, test_remove) { MetricSet parent("parent", {}, ""); SumMetric<LongCountMetric> sum("foo", {}, "foodesc", &parent); @@ -116,13 +96,12 @@ SumMetricTest::testRemove() v2.inc(7); v3.inc(10); - CPPUNIT_ASSERT_EQUAL(int64_t(20), sum.getLongValue("value")); + EXPECT_EQ(int64_t(20), sum.getLongValue("value")); sum.removeMetricFromSum(v2); - CPPUNIT_ASSERT_EQUAL(int64_t(13), sum.getLongValue("value")); + EXPECT_EQ(int64_t(13), sum.getLongValue("value")); } -void -SumMetricTest::testStartValue() +TEST(SumMetricTest, test_start_value) { MetricSnapshot snapshot("active"); SumMetric<LongValueMetric> sum("foo", {}, "foodesc", @@ -132,7 +111,7 @@ SumMetricTest::testStartValue() sum.setStartValue(start); // without children - CPPUNIT_ASSERT_EQUAL(int64_t(50), sum.getLongValue("value")); + EXPECT_EQ(int64_t(50), sum.getLongValue("value")); MetricSnapshot copy("copy"); copy.recreateSnapshot(snapshot.getMetrics(), true); @@ -143,7 +122,7 @@ SumMetricTest::testStartValue() value.set(10); // with children - CPPUNIT_ASSERT_EQUAL(int64_t(60), sum.getLongValue("value")); + EXPECT_EQ(int64_t(60), sum.getLongValue("value")); } -} // metrics +} diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java index a469e666d93..419bc7ddf28 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java @@ -20,6 +20,7 @@ import com.yahoo.tensor.functions.ScalarFunctions; import onnx.Onnx; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -33,8 +34,8 @@ class GraphImporter { private static IntermediateOperation mapOperation(Onnx.NodeProto node, List<IntermediateOperation> inputs, IntermediateGraph graph) { - String nodeName = node.getName(); String modelName = graph.name(); + String nodeName = getNodeName(node); switch (node.getOpType().toLowerCase()) { case "abs": return new Map(modelName, nodeName, inputs, ScalarFunctions.abs()); @@ -74,7 +75,7 @@ class GraphImporter { case "tanh": return new Map(modelName, nodeName, inputs, ScalarFunctions.tanh()); } - IntermediateOperation op = new NoOp(modelName, node.getName(), inputs); + IntermediateOperation op = new NoOp(modelName, nodeName, inputs); op.warning("Operation '" + node.getOpType() + "' is currently not implemented"); return op; } @@ -199,18 +200,47 @@ class GraphImporter { } private static Onnx.NodeProto getNodeFromGraph(String nodeName, Onnx.GraphProto graph) { - boolean hasPortNumber = nodeName.contains(":"); + Optional<Onnx.NodeProto> node; + if (nodeName.contains(":")) { + node = getNodeFromGraphOutputs(nodeName, graph); + } else { + node = getNodeFromGraphNames(nodeName, graph); + if (node.isEmpty()) { + node = getNodeFromGraphOutputs(nodeName, graph); + } + } + return node.orElseThrow(() -> new IllegalArgumentException("Node '" + nodeName + "' not found in ONNX graph")); + } + + private static Optional<Onnx.NodeProto> getNodeFromGraphOutputs(String nodeName, Onnx.GraphProto graph) { for (Onnx.NodeProto node : graph.getNodeList()) { - if (hasPortNumber) { - for (String outputName : node.getOutputList()) { - if (outputName.equals(nodeName)) { - return node; - } + for (String outputName : node.getOutputList()) { + if (outputName.equals(nodeName)) { + return Optional.of(node); } - } else if (node.getName().equals(nodeName)) { - return node; } } - throw new IllegalArgumentException("Node '" + nodeName + "' not found in ONNX graph"); + return Optional.empty(); + } + + private static Optional<Onnx.NodeProto> getNodeFromGraphNames(String nodeName, Onnx.GraphProto graph) { + for (Onnx.NodeProto node : graph.getNodeList()) { + if (node.getName().equals(nodeName)) { + return Optional.of(node); + } + } + return Optional.empty(); + } + + private static String getNodeName(Onnx.NodeProto node) { + String nodeName = node.getName(); + if (nodeName.length() > 0) + return nodeName; + if (node.getOutputCount() == 1) + return node.getOutput(0); + throw new IllegalArgumentException("Unable to find a suitable name for node '" + node.toString() + "'. " + + "Either no explicit name given or no single output name."); } + + } diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java new file mode 100644 index 00000000000..d1dea730da5 --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package ai.vespa.rankingexpression.importer.onnx; + +import ai.vespa.rankingexpression.importer.ImportedModel; +import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author lesters + */ +public class SimpleImportTestCase { + + @Test + public void testSimpleOnnxModelImport() { + ImportedModel model = new OnnxImporter().importModel("test", "src/test/models/onnx/simple/simple.onnx"); + + MapContext context = new MapContext(); + context.put("query_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[1],d1[4])")). + cell(0.1, 0, 0). + cell(0.2, 0, 1). + cell(0.3, 0, 2). + cell(0.4, 0, 3).build())); + context.put("attribute_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[4],d1[1])")). + cell(0.1, 0, 0). + cell(0.2, 1, 0). + cell(0.3, 2, 0). + cell(0.4, 3, 0).build())); + context.put("bias_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[1],d1[1])")). + cell(1.0, 0, 0).build())); + + Tensor result = model.expressions().get("output").evaluate(context).asTensor(); + assertEquals(result, Tensor.from("tensor(d0[1],d1[1]):{{d0:0,d1:0}:1.3}")); + } + +} diff --git a/model-integration/src/test/models/onnx/simple/simple.onnx b/model-integration/src/test/models/onnx/simple/simple.onnx new file mode 100644 index 00000000000..1c746c90efa --- /dev/null +++ b/model-integration/src/test/models/onnx/simple/simple.onnx @@ -0,0 +1,23 @@ + simple.py:ã +0 + query_tensor +attribute_tensormatmul"MatMul +" +matmul + bias_tensoroutput"addsimple_scoringZ + query_tensor + + +Z" +attribute_tensor + + +Z + bias_tensor + + +b +output + + +B diff --git a/model-integration/src/test/models/onnx/simple/simple.py b/model-integration/src/test/models/onnx/simple/simple.py new file mode 100755 index 00000000000..4471ed812b8 --- /dev/null +++ b/model-integration/src/test/models/onnx/simple/simple.py @@ -0,0 +1,32 @@ +import onnx +from onnx import helper, TensorProto + +QUERY_TENSOR = helper.make_tensor_value_info('query_tensor', TensorProto.FLOAT, [1, 4]) +ATTRIBUTE_TENSOR = helper.make_tensor_value_info('attribute_tensor', TensorProto.FLOAT, [4, 1]) +BIAS_TENSOR = helper.make_tensor_value_info('bias_tensor', TensorProto.FLOAT, [1, 1]) +OUTPUT = helper.make_tensor_value_info('output', TensorProto.FLOAT, [1, 1]) + +nodes = [ + helper.make_node( + 'MatMul', + ['query_tensor', 'attribute_tensor'], + ['matmul'], + ), + helper.make_node( + 'add', + ['matmul', 'bias_tensor'], + ['output'], + ), +] +graph_def = helper.make_graph( + nodes, + 'simple_scoring', + [ + QUERY_TENSOR, + ATTRIBUTE_TENSOR, + BIAS_TENSOR, + ], + [OUTPUT], +) +model_def = helper.make_model(graph_def, producer_name='simple.py') +onnx.save(model_def, 'simple.onnx') @@ -96,7 +96,6 @@ <module>messagebus-disc</module> <module>messagebus</module> <module>metrics</module> - <module>metrics-proxy</module> <module>model-evaluation</module> <module>model-integration</module> <module>node-repository</module> diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index b091ad569fa..46ee5e15757 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -299,11 +299,10 @@ writeField(FakeWordSet &wordSet, ostate.open(); unsigned int wordNum = 1; - for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) { - for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) { - FakeWord &fw = *wordSet._words[wc][wi]; + for (const auto& words : wordSet.words()) { + for (const auto& word : words) { ostate._fieldWriter->newWord(makeWordString(wordNum)); - fw.dump(*ostate._fieldWriter, false); + word->dump(*ostate._fieldWriter, false); ++wordNum; } } @@ -349,15 +348,12 @@ readField(FakeWordSet &wordSet, TermFieldMatchData mdfield1; unsigned int wordNum = 1; - for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) { - for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) { - FakeWord &fw = *wordSet._words[wc][wi]; - + for (const auto& words : wordSet.words()) { + for (const auto& word : words) { TermFieldMatchDataArray tfmda; tfmda.add(&mdfield1); - fw.validate(*istate._fieldReader, wordNum, - tfmda, verbose); + word->validate(*istate._fieldReader, wordNum, tfmda, verbose); ++wordNum; } } @@ -425,10 +421,8 @@ randReadField(FakeWordSet &wordSet, for (int loop = 0; loop < 1; ++loop) { unsigned int wordNum = 1; - for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) { - for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) { - FakeWord &fw = *wordSet._words[wc][wi]; - + for (const auto& words : wordSet.words()) { + for (const auto& word : words) { PostingListOffsetAndCounts offsetAndCounts; uint64_t checkWordNum; dictFile->lookup(makeWordString(wordNum), @@ -456,12 +450,12 @@ randReadField(FakeWordSet &wordSet, sb(handle.createIterator(counts, tfmda)); // LOG(info, "loop=%d, wordNum=%u", loop, wordNum); - fw.validate(sb.get(), tfmda, verbose); - fw.validate(sb.get(), tfmda, 19, verbose); - fw.validate(sb.get(), tfmda, 99, verbose); - fw.validate(sb.get(), tfmda, 799, verbose); - fw.validate(sb.get(), tfmda, 6399, verbose); - fw.validate(sb.get(), tfmda, 11999, verbose); + word->validate(sb.get(), tfmda, verbose); + word->validate(sb.get(), tfmda, 19, verbose); + word->validate(sb.get(), tfmda, 99, verbose); + word->validate(sb.get(), tfmda, 799, verbose); + word->validate(sb.get(), tfmda, 6399, verbose); + word->validate(sb.get(), tfmda, 11999, verbose); ++wordNum; } } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index f08cb0855af..1d9433c739f 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -1217,8 +1217,8 @@ Test::testDotProduct() vespalib::Stash stash; FeatureExecutor &exc = bp.createExecutor(ft.getQueryEnv(), stash); // check that we have the optimized enum version - dotproduct::wset::DotProductExecutor<dotproduct::wset::EnumVector, WeightedEnumContent> * myExc = - dynamic_cast<dotproduct::wset::DotProductExecutor<dotproduct::wset::EnumVector, WeightedEnumContent> *>(&exc); + dotproduct::wset::DotProductExecutorByCopy<dotproduct::wset::EnumVector, WeightedEnumContent> * myExc = + dynamic_cast<dotproduct::wset::DotProductExecutorByCopy<dotproduct::wset::EnumVector, WeightedEnumContent> *>(&exc); EXPECT_TRUE(myExc != nullptr); EXPECT_EQUAL(1u, deps.output.size()); } diff --git a/searchlib/src/tests/postinglistbm/CMakeLists.txt b/searchlib/src/tests/postinglistbm/CMakeLists.txt index 2fb507dbd01..479f54e8531 100644 --- a/searchlib/src/tests/postinglistbm/CMakeLists.txt +++ b/searchlib/src/tests/postinglistbm/CMakeLists.txt @@ -1,4 +1,14 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_posting_list_test_app TEST + SOURCES + posting_list_test.cpp + DEPENDS + searchlib_test + searchlib + gtest +) +vespa_add_test(NAME searchlib_posting_list_test_app NO_VALGRIND COMMAND searchlib_posting_list_test_app) + vespa_add_executable(searchlib_postinglistbm_app TEST SOURCES postinglistbm.cpp @@ -7,4 +17,3 @@ vespa_add_executable(searchlib_postinglistbm_app TEST searchlib_test searchlib ) -vespa_add_test(NAME searchlib_postinglistbm_app NO_VALGRIND COMMAND searchlib_postinglistbm_app -q -a) diff --git a/searchlib/src/tests/postinglistbm/andstress.cpp b/searchlib/src/tests/postinglistbm/andstress.cpp index 7152e2a3981..adca7892464 100644 --- a/searchlib/src/tests/postinglistbm/andstress.cpp +++ b/searchlib/src/tests/postinglistbm/andstress.cpp @@ -192,9 +192,9 @@ AndStressMaster::resetTasks() _taskIdx = 0; } -static void +void makeSomePostings(FPFactory *postingFactory, - std::vector<FakeWord *> &words, + const FakeWordSet::FakeWordVector &words, std::vector<FakePosting::SP> &postings, uint32_t stride, bool validate, @@ -234,9 +234,9 @@ AndStressMaster::makePostingsHelper(FPFactory *postingFactory, tv.SetNow(); before = tv.Secs(); postingFactory->setup(_wordSet); - for (size_t i = 0; i < _wordSet._words.size(); ++i) + for (size_t i = 0; i < _wordSet.words().size(); ++i) makeSomePostings(postingFactory, - _wordSet._words[i], _postings[i], + _wordSet.words()[i], _postings[i], _stride, validate, verbose); diff --git a/searchlib/src/tests/postinglistbm/posting_list_test.cpp b/searchlib/src/tests/postinglistbm/posting_list_test.cpp new file mode 100644 index 00000000000..ad3410b8f92 --- /dev/null +++ b/searchlib/src/tests/postinglistbm/posting_list_test.cpp @@ -0,0 +1,173 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/test/fakedata/fake_match_loop.h> +#include <vespa/searchlib/test/fakedata/fakeposting.h> +#include <vespa/searchlib/test/fakedata/fakeword.h> +#include <vespa/searchlib/test/fakedata/fakewordset.h> +#include <vespa/searchlib/test/fakedata/fpfactory.h> +#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <cinttypes> + +using search::fef::TermFieldMatchData; +using search::fef::TermFieldMatchDataArray; +using search::queryeval::SearchIterator; + +using namespace search::index; +using namespace search::fakedata; + +using FakeWordUP = std::unique_ptr<FakeWord>; + +void +validate_posting_list_for_word(const FakePosting& posting, const FakeWord& word) +{ + TermFieldMatchData md; + TermFieldMatchDataArray tfmda; + tfmda.add(&md); + + std::unique_ptr<SearchIterator> iterator(posting.createIterator(tfmda)); + if (posting.hasWordPositions()) { + word.validate(iterator.get(), tfmda, false); + } else { + word.validate(iterator.get(), false); + } +} + +void +test_fake(const std::string& posting_type, + const Schema& schema, + const FakeWord& word) +{ + std::unique_ptr<FPFactory> factory(getFPFactory(posting_type, schema)); + std::vector<const FakeWord *> words; + words.push_back(&word); + factory->setup(words); + auto posting = factory->make(word); + + printf("%s.bitsize=%d+%d+%d+%d+%d\n", + posting->getName().c_str(), + static_cast<int>(posting->bitSize()), + static_cast<int>(posting->l1SkipBitSize()), + static_cast<int>(posting->l2SkipBitSize()), + static_cast<int>(posting->l3SkipBitSize()), + static_cast<int>(posting->l4SkipBitSize())); + + validate_posting_list_for_word(*posting, word); + + uint64_t scan_time = 0; + uint64_t scan_unpack_time = 0; + int hits1 = FakeMatchLoop::single_posting_scan(*posting, word.getDocIdLimit(), scan_time); + int hits2 = FakeMatchLoop::single_posting_scan_with_unpack(*posting, word.getDocIdLimit(), scan_unpack_time); + + printf("test_fake: '%s': hits1=%d, hits2=%d, scan_time=%" PRIu64 "(ns), scan_unpack_time=%" PRIu64 "(ns)\n", + posting->getName().c_str(), hits1, hits2, scan_time, scan_unpack_time); +} + +void +test_fake_pair(const std::string& posting_type, const Schema& schema, + const FakeWord& word1, const FakeWord& word2) +{ + std::unique_ptr<FPFactory> factory(getFPFactory(posting_type, schema)); + std::vector<const FakeWord *> words; + words.push_back(&word1); + words.push_back(&word2); + factory->setup(words); + auto posting1 = factory->make(word1); + auto posting2 = factory->make(word2); + + uint64_t scan_time = 0; + int hits = FakeMatchLoop::and_pair_posting_scan(*posting1, *posting2, word1.getDocIdLimit(), scan_time); + printf("test_fake_pair: '%s' AND '%s' => %d hits, scan_time=%" PRIu64 " (ns)\n", + posting1->getName().c_str(), posting2->getName().c_str(), hits, scan_time); +} + +struct PostingListTest : public ::testing::Test { + uint32_t num_docs; + std::vector<std::string> posting_types; + FakeWordSet word_set; + FakeWordUP word1; + FakeWordUP word2; + FakeWordUP word3; + FakeWordUP word4; + FakeWordUP word5; + search::Rand48 rnd; + + PostingListTest() + : num_docs(36000), + posting_types(getPostingTypes()), + word_set(), + word1(), + word2(), + word3(), + word4(), + word5(), + rnd() + { + rnd.srand48(32); + } + + void setup(bool has_elements, bool has_element_weights) { + word_set.setupParams(has_elements, has_element_weights); + + uint32_t w1_freq = 2; + uint32_t w2_freq = 1000; + uint32_t w3_freq = 10000; + uint32_t w4_freq = 19000; + uint32_t w5_freq = 5000; + uint32_t w4w5od = 1000; + + word1 = std::make_unique<FakeWord>(num_docs, w1_freq, w1_freq / 2, "word1", rnd, + word_set.getFieldsParams(), word_set.getPackedIndex()); + + word2 = std::make_unique<FakeWord>(num_docs, w2_freq, w2_freq / 2, "word2", *word1, 4, rnd, + word_set.getFieldsParams(), word_set.getPackedIndex()); + + word3 = std::make_unique<FakeWord>(num_docs, w3_freq, w3_freq / 2, "word3", *word1, 10, rnd, + word_set.getFieldsParams(), word_set.getPackedIndex()); + + word4 = std::make_unique<FakeWord>(num_docs, w4_freq, w4_freq / 2, "word4", rnd, + word_set.getFieldsParams(), word_set.getPackedIndex()); + + word5 = std::make_unique<FakeWord>(num_docs, w5_freq, w5_freq / 2, "word5", *word4, w4w5od, rnd, + word_set.getFieldsParams(), word_set.getPackedIndex()); + + } + + void run() { + for (const auto& type : posting_types) { + test_fake(type, word_set.getSchema(), *word1); + test_fake(type, word_set.getSchema(), *word2); + test_fake(type, word_set.getSchema(), *word3); + } + + for (const auto& type : posting_types) { + test_fake_pair(type, word_set.getSchema(), *word1, *word3); + test_fake_pair(type, word_set.getSchema(), *word2, *word3); + } + + for (const auto& type : posting_types) { + test_fake_pair(type, word_set.getSchema(), *word4, *word5); + } + } + +}; + +TEST_F(PostingListTest, verify_posting_list_iterators_over_single_value_field) +{ + setup(false, false); + run(); +} + +TEST_F(PostingListTest, verify_posting_list_iterators_over_array_field) +{ + setup(true, false); + run(); +} + +TEST_F(PostingListTest, verify_posting_list_iterators_over_weighted_set_field) +{ + setup(true, true); + run(); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/postinglistbm/postinglistbm.cpp b/searchlib/src/tests/postinglistbm/postinglistbm.cpp index 02fbe4878ba..0a6f99ede11 100644 --- a/searchlib/src/tests/postinglistbm/postinglistbm.cpp +++ b/searchlib/src/tests/postinglistbm/postinglistbm.cpp @@ -24,21 +24,16 @@ using search::queryeval::SearchIterator; using namespace search::index; using namespace search::fakedata; -// needed to resolve external symbol from httpd.h on AIX -void FastS_block_usr2() {} - - namespace postinglistbm { class PostingListBM : public FastOS_Application { private: - bool _verbose; uint32_t _numDocs; uint32_t _commonDocFreq; uint32_t _numWordsPerClass; std::vector<std::string> _postingTypes; uint32_t _loops; - unsigned int _skipCommonPairsRate; + uint32_t _skipCommonPairsRate; FakeWordSet _wordSet; uint32_t _stride; bool _unpack; @@ -46,12 +41,6 @@ private: public: search::Rand48 _rnd; -private: - void usage(); - void badPostingType(const std::string &postingType); - void testFake(const std::string &postingType, - const Schema &schema, - const FakeWord &word); public: PostingListBM(); ~PostingListBM(); @@ -59,23 +48,23 @@ public: }; void -PostingListBM::usage() +usage() { - printf("postinglistbm " + printf("Usage: postinglistbm " "[-C <skipCommonPairsRate>] " - "[-a] " + "[-T {string, array, weightedSet}] " "[-c <commonDoqFreq>] " "[-d <numDocs>] " "[-l <numLoops>] " "[-s <stride>] " "[-t <postingType>] " "[-u] " - "[-q] " - "[-v]\n"); + "[-w <numWordsPerClass>] " + "[-q]\n"); } void -PostingListBM::badPostingType(const std::string &postingType) +badPostingType(const std::string &postingType) { printf("Bad posting list type: %s\n", postingType.c_str()); printf("Supported types: "); @@ -93,8 +82,7 @@ PostingListBM::badPostingType(const std::string &postingType) } PostingListBM::PostingListBM() - : _verbose(false), - _numDocs(10000000), + : _numDocs(10000000), _commonDocFreq(50000), _numWordsPerClass(100), _postingTypes(), @@ -109,93 +97,19 @@ PostingListBM::PostingListBM() PostingListBM::~PostingListBM() = default; -void -validate_posting_for_word(const FakePosting& posting, const FakeWord& word, bool verbose) -{ - TermFieldMatchData md; - TermFieldMatchDataArray tfmda; - tfmda.add(&md); - - std::unique_ptr<SearchIterator> iterator(posting.createIterator(tfmda)); - if (posting.hasWordPositions()) { - word.validate(iterator.get(), tfmda, verbose); - } else { - word.validate(iterator.get(), verbose); - } -} - -void -PostingListBM::testFake(const std::string &postingType, - const Schema &schema, - const FakeWord &word) -{ - auto posting_factory = getFPFactory(postingType, schema); - std::vector<const FakeWord *> words; - words.push_back(&word); - posting_factory->setup(words); - auto posting = posting_factory->make(word); - - printf("%s.bitsize=%d+%d+%d+%d+%d\n", - posting->getName().c_str(), - static_cast<int>(posting->bitSize()), - static_cast<int>(posting->l1SkipBitSize()), - static_cast<int>(posting->l2SkipBitSize()), - static_cast<int>(posting->l3SkipBitSize()), - static_cast<int>(posting->l4SkipBitSize())); - - validate_posting_for_word(*posting, word, _verbose); - - uint64_t scanTime = 0; - uint64_t scanUnpackTime = 0; - int hits1 = FakeMatchLoop::single_posting_scan(*posting, word.getDocIdLimit(), scanTime); - int hits2 = FakeMatchLoop::single_posting_scan_with_unpack(*posting, word.getDocIdLimit(), scanUnpackTime); - - printf("testFake '%s' hits1=%d, hits2=%d, scanTime=%" PRIu64 - ", scanUnpackTime=%" PRIu64 "\n", - posting->getName().c_str(), - hits1, hits2, scanTime, scanUnpackTime); -} - -void -testFakePair(const std::string &postingType, - const Schema &schema, - bool unpack, - const FakeWord &fw1, const FakeWord &fw2) -{ - std::unique_ptr<FPFactory> ff(getFPFactory(postingType, schema)); - std::vector<const FakeWord *> v; - v.push_back(&fw1); - v.push_back(&fw2); - ff->setup(v); - FakePosting::SP f1(ff->make(fw1)); - FakePosting::SP f2(ff->make(fw2)); - - uint64_t scanUnpackTime = 0; - int hits = unpack ? - FakeMatchLoop::and_pair_posting_scan_with_unpack(*f1, *f2, fw1.getDocIdLimit(), scanUnpackTime) : - FakeMatchLoop::and_pair_posting_scan(*f1, *f2, fw1.getDocIdLimit(), scanUnpackTime); - printf("Fakepair %s AND %s => %d hits, %" PRIu64 " cycles\n", - f1->getName().c_str(), - f2->getName().c_str(), - hits, - scanUnpackTime); -} - int PostingListBM::Main() { int argi; char c; const char *optArg; - bool doandstress; - doandstress = false; argi = 1; bool hasElements = false; bool hasElementWeights = false; bool quick = false; - while ((c = GetOpt("C:ac:d:l:s:t:uvw:T:q", optArg, argi)) != -1) { + while ((c = GetOpt("C:c:d:l:s:t:uw:T:q", optArg, argi)) != -1) { switch(c) { case 'C': _skipCommonPairsRate = atoi(optArg); @@ -215,9 +129,6 @@ PostingListBM::Main() return 1; } break; - case 'a': - doandstress = true; - break; case 'c': _commonDocFreq = atoi(optArg); break; @@ -248,9 +159,6 @@ PostingListBM::Main() case 'u': _unpack = true; break; - case 'v': - _verbose = true; - break; case 'w': _numWordsPerClass = atoi(optArg); break; @@ -273,65 +181,24 @@ PostingListBM::Main() _wordSet.setupParams(hasElements, hasElementWeights); - uint32_t w1dfreq = 10; - uint32_t w4dfreq = 790000; - uint32_t w5dfreq = 290000; - uint32_t w4w5od = 100000; uint32_t numTasks = 40000; if (quick) { - w1dfreq = 2; - w4dfreq = 19000; - w5dfreq = 5000; - w4w5od = 1000; numTasks = 40; } - - FakeWord word1(_numDocs, w1dfreq, w1dfreq / 2, "word1", _rnd, - _wordSet.getFieldsParams(), _wordSet.getPackedIndex()); - FakeWord word2(_numDocs, 1000, 500, "word2", word1, 4, _rnd, - _wordSet.getFieldsParams(), _wordSet.getPackedIndex()); - FakeWord word3(_numDocs, _commonDocFreq, _commonDocFreq / 2, - "word3", word1, 10, _rnd, - _wordSet.getFieldsParams(), _wordSet.getPackedIndex()); - FakeWord word4(_numDocs, w4dfreq, w4dfreq / 2, - "word4", _rnd, - _wordSet.getFieldsParams(), _wordSet.getPackedIndex()); - FakeWord word5(_numDocs, w5dfreq, w5dfreq / 2, - "word5", word4, w4w5od, _rnd, - _wordSet.getFieldsParams(), _wordSet.getPackedIndex()); - if (_postingTypes.empty()) { _postingTypes = getPostingTypes(); } - for (const auto& type : _postingTypes) { - testFake(type, _wordSet.getSchema(), word1); - testFake(type, _wordSet.getSchema(), word2); - testFake(type, _wordSet.getSchema(), word3); - } - - for (const auto& type : _postingTypes) { - testFakePair(type, _wordSet.getSchema(), false, word1, word3); - testFakePair(type, _wordSet.getSchema(), false, word2, word3); - } - - for (const auto& type : _postingTypes) { - testFakePair(type, _wordSet.getSchema(), false, word4, word5); - } + _wordSet.setupWords(_rnd, _numDocs, _commonDocFreq, _numWordsPerClass); - if (doandstress) { - _wordSet.setupWords(_rnd, _numDocs, _commonDocFreq, _numWordsPerClass); - } - if (doandstress) { - AndStress andstress; - andstress.run(_rnd, _wordSet, - _numDocs, _commonDocFreq, _postingTypes, _loops, - _skipCommonPairsRate, - numTasks, - _stride, - _unpack); - } + AndStress andstress; + andstress.run(_rnd, _wordSet, + _numDocs, _commonDocFreq, _postingTypes, _loops, + _skipCommonPairsRate, + numTasks, + _stride, + _unpack); return 0; } diff --git a/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp b/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp index 652ef2e0889..4e637fe19cf 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp @@ -46,6 +46,23 @@ void encode_message(const MSG &src, FRT_Values &dst) { dst.AddData(compressed.getData(), compressed.getDataLen()); } +void encode_search_reply(const ProtoSearchReply &src, FRT_Values &dst) { + using vespalib::compression::compress; + auto output = src.SerializeAsString(); + if (src.grouping_blob().empty()) { + dst.AddInt8(CompressionConfig::Type::NONE); + dst.AddInt32(output.size()); + dst.AddData(output.data(), output.size()); + } else { + ConstBufferRef buf(output.data(), output.size()); + DataBuffer compressed(output.data(), output.size()); + CompressionConfig::Type type = compress(get_compression_config(), buf, compressed, true); + dst.AddInt8(type); + dst.AddInt32(buf.size()); + dst.AddData(compressed.getData(), compressed.getDataLen()); + } +} + template <typename MSG> bool decode_message(const FRT_Values &src, MSG &dst) { using vespalib::compression::decompress; @@ -88,7 +105,7 @@ struct SearchCompletionHandler : SearchClient { void searchDone(SearchReply::UP reply) override { ProtoSearchReply msg; ProtoConverter::search_reply_to_proto(*reply, msg); - encode_message(msg, *req.GetReturn()); + encode_search_reply(msg, *req.GetReturn()); req.Return(); } }; diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 1dcd3e35580..55a550837e1 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -30,27 +30,38 @@ VectorBase<DimensionVType, DimensionHType, ComponentType, HashMapComparator>::Ve template <typename DimensionVType, typename DimensionHType, typename ComponentType, typename HashMapComparator> VectorBase<DimensionVType, DimensionHType, ComponentType, HashMapComparator>::~VectorBase() = default; -template <typename V> -V copyAndSync(const V & v) { - V tmp(v); - tmp.syncMap(); - return tmp; +template <typename DimensionVType, typename DimensionHType, typename ComponentType, typename HashMapComparator> +VectorBase<DimensionVType, DimensionHType, ComponentType, HashMapComparator> & +VectorBase<DimensionVType, DimensionHType, ComponentType, HashMapComparator>::syncMap() { + Converter<DimensionVType, DimensionHType> conv; + _dimMap.clear(); + _dimMap.resize(_vector.size()*2); + for (size_t i = 0; i < _vector.size(); ++i) { + _dimMap.insert(std::make_pair(conv.convert(_vector[i].first), _vector[i].second)); + } + return *this; } +template VectorBase<int64_t, int64_t, double> & VectorBase<int64_t, int64_t, double>::syncMap(); + + template <typename Vector, typename Buffer> -DotProductExecutor<Vector, Buffer>::DotProductExecutor(const IAttributeVector * attribute, const Vector & queryVector) : +DotProductExecutorByCopy<Vector, Buffer>::DotProductExecutorByCopy(const IAttributeVector * attribute, Vector queryVector) : FeatureExecutor(), _attribute(attribute), - _queryVector(copyAndSync(queryVector)), - _end(_queryVector.getDimMap().end()), + _queryVector(std::move(queryVector)), + _end(_queryVector.syncMap().getDimMap().end()), _buffer() { _buffer.allocate(_attribute->getMaxValueCount()); } template <typename Vector, typename Buffer> +DotProductExecutorByCopy<Vector, Buffer>::~DotProductExecutorByCopy() = default; + +template <typename Vector, typename Buffer> void -DotProductExecutor<Vector, Buffer>::execute(uint32_t docId) +DotProductExecutorByCopy<Vector, Buffer>::execute(uint32_t docId) { feature_t val = 0; if (!_queryVector.getDimMap().empty()) { @@ -69,6 +80,50 @@ StringVector::StringVector() = default; StringVector::~StringVector() = default; +template <typename BaseType> +DotProductExecutorBase<BaseType>::DotProductExecutorBase(V queryVector) + : FeatureExecutor(), + _queryVector(std::move(queryVector)), + _end(_queryVector.syncMap().getDimMap().end()) +{ +} + +template <typename BaseType> +DotProductExecutorBase<BaseType>::~DotProductExecutorBase() = default; + +template <typename BaseType> +void DotProductExecutorBase<BaseType>::execute(uint32_t docId) { + feature_t val = 0; + if (!_queryVector.getDimMap().empty()) { + const AT * values(nullptr); + uint32_t sz = getAttributeValues(docId, values); + for (size_t i = 0; i < sz; ++i) { + typename V::HashMap::const_iterator itr = _queryVector.getDimMap().find(values[i].value()); + if (itr != _end) { + val += values[i].weight() * itr->second; + } + } + } + outputs().set_number(0, val); +} + +template <typename A> +DotProductExecutor<A>::DotProductExecutor(const A * attribute, V queryVector) : + DotProductExecutorBase<typename A::BaseType>(std::move(queryVector)), + _attribute(attribute) +{ +} + +template <typename A> +DotProductExecutor<A>::~DotProductExecutor() = default; + +template <typename A> +size_t +DotProductExecutor<A>::getAttributeValues(uint32_t docId, const AT * & values) +{ + return _attribute->getRawValues(docId, values); +} + } namespace dotproduct::array { @@ -507,9 +562,8 @@ createFromObject(const IAttributeVector * attribute, const fef::Anything & objec return stash.create<SingleZeroValueExecutor>(); } -FeatureExecutor * createTypedArrayExecutor(const IAttributeVector * attribute, - const Property & prop, - vespalib::Stash & stash) { +FeatureExecutor * +createTypedArrayExecutor(const IAttributeVector * attribute, const Property & prop, vespalib::Stash & stash) { if (!attribute->isImported()) { switch (attribute->getBasicType()) { case BasicType::INT32: @@ -542,29 +596,55 @@ FeatureExecutor * createTypedArrayExecutor(const IAttributeVector * attribute, return nullptr; } -FeatureExecutor * createTypedWsetExecutor(const IAttributeVector * attribute, - const Property & prop, - vespalib::Stash & stash) { - if (attribute->isStringType()) { - if (attribute->hasEnum()) { - dotproduct::wset::EnumVector vector(attribute); - WeightedSetParser::parse(prop.get(), vector); - return &stash.create<dotproduct::wset::DotProductExecutor<dotproduct::wset::EnumVector, WeightedEnumContent>>(attribute, vector); - } else { - dotproduct::wset::StringVector vector; - WeightedSetParser::parse(prop.get(), vector); - return &stash.create<dotproduct::wset::DotProductExecutor<dotproduct::wset::StringVector, WeightedConstCharContent>>(attribute, vector); +template <typename A, typename V> +FeatureExecutor * +createForDirectWSetImpl(const IAttributeVector * attribute, V vector, vespalib::Stash & stash) +{ + using namespace dotproduct::wset; + using T = typename A::BaseType; + const A * iattr = dynamic_cast<const A *>(attribute); + if (!attribute->isImported() && (iattr != nullptr) && supportsGetRawValues(*iattr)) { + using VT = multivalue::WeightedValue<T>; + using ExactA = MultiValueNumericAttribute<A, VT>; + + const ExactA * exactA = dynamic_cast<const ExactA *>(iattr); + if (exactA != nullptr) { + return &stash.create<DotProductExecutor<ExactA>>(exactA, std::move(vector)); } - } else if (attribute->isIntegerType()) { - if (attribute->hasEnum()) { - dotproduct::wset::EnumVector vector(attribute); - WeightedSetParser::parse(prop.get(), vector); - return &stash.create<dotproduct::wset::DotProductExecutor<dotproduct::wset::EnumVector, WeightedEnumContent>>(attribute, vector); + return &stash.create<DotProductExecutor<A>>(iattr, std::move(vector)); + } + return &stash.create<DotProductExecutorByCopy<IntegerVectorT<T>, WeightedIntegerContent>>(attribute, std::move(vector)); +} - } else { - dotproduct::wset::IntegerVector vector; +template <typename T> +FeatureExecutor * +createForDirectIntegerWSet(const IAttributeVector * attribute, const Property & prop, vespalib::Stash & stash) +{ + using namespace dotproduct::wset; + IntegerVectorT<T> vector; + WeightedSetParser::parse(prop.get(), vector); + return createForDirectWSetImpl<IntegerAttributeTemplate<T>>(attribute, std::move(vector), stash); +} + + +FeatureExecutor * +createTypedWsetExecutor(const IAttributeVector * attribute, const Property & prop, vespalib::Stash & stash) { + using namespace dotproduct::wset; + if (attribute->hasEnum()) { + EnumVector vector(attribute); + WeightedSetParser::parse(prop.get(), vector); + return &stash.create<DotProductExecutorByCopy<EnumVector, WeightedEnumContent>>(attribute, std::move(vector)); + } else { + if (attribute->isStringType()) { + StringVector vector; WeightedSetParser::parse(prop.get(), vector); - return &stash.create<dotproduct::wset::DotProductExecutor<dotproduct::wset::IntegerVector, WeightedIntegerContent>>(attribute, vector); + return &stash.create<DotProductExecutorByCopy<StringVector, WeightedConstCharContent>>(attribute, std::move(vector)); + } else if (attribute->isIntegerType()) { + if (attribute->getBasicType() == BasicType::INT32) { + return createForDirectIntegerWSet<int32_t>(attribute, prop, stash); + } else if (attribute->getBasicType() == BasicType::INT64) { + return createForDirectIntegerWSet<int64_t>(attribute, prop, stash); + } } } return nullptr; diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h index 089066cb5f6..38dcdd54929 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h @@ -55,71 +55,106 @@ protected: Vector _vector; HashMap _dimMap; // dimension -> component public: + VectorBase(VectorBase && rhs) = default; + VectorBase & operator = (VectorBase && rhs) = default; ~VectorBase(); const Vector & getVector() const { return _vector; } - void syncMap() { - Converter<DimensionVType, DimensionHType> conv; - _dimMap.clear(); - _dimMap.resize(_vector.size()*2); - for (size_t i = 0; i < _vector.size(); ++i) { - _dimMap.insert(std::make_pair(conv.convert(_vector[i].first), _vector[i].second)); - } - } + VectorBase & syncMap(); const HashMap & getDimMap() const { return _dimMap; } }; /** * Represents a vector where the dimensions are integers. **/ -class IntegerVector : public VectorBase<int64_t, int64_t, feature_t> { +template<typename T> +class IntegerVectorT : public VectorBase<T, T, feature_t> { public: void insert(vespalib::stringref label, vespalib::stringref value) { - _vector.push_back(std::make_pair(util::strToNum<int64_t>(label), util::strToNum<feature_t>(value))); + this->_vector.emplace_back(util::strToNum<T>(label), util::strToNum<feature_t>(value)); } }; +using IntegerVector = IntegerVectorT<int64_t>; + /** * Represents a vector where the dimensions are string values. **/ class StringVector : public VectorBase<vespalib::string, const char *, feature_t, ConstCharComparator> { public: StringVector(); + StringVector(StringVector &&) = default; + StringVector & operator = (StringVector &&) = default; ~StringVector(); void insert(vespalib::stringref label, vespalib::stringref value) { - _vector.push_back(std::make_pair(label, util::strToNum<feature_t>(value))); + _vector.emplace_back(label, util::strToNum<feature_t>(value)); } }; /** * Represents a vector where the dimensions are enum values for strings. **/ -class EnumVector : public VectorBase<search::attribute::EnumHandle, search::attribute::EnumHandle, feature_t> { +class EnumVector : public VectorBase<attribute::EnumHandle, attribute::EnumHandle, feature_t> { private: const attribute::IAttributeVector * _attribute; public: EnumVector(const attribute::IAttributeVector * attribute) : _attribute(attribute) {} void insert(vespalib::stringref label, vespalib::stringref value) { - search::attribute::EnumHandle e; + attribute::EnumHandle e; if (_attribute->findEnum(label.data(), e)) { - _vector.push_back(std::make_pair(e, util::strToNum<feature_t>(value))); + _vector.emplace_back(e, util::strToNum<feature_t>(value)); } } }; +/** + * Common base for handling execution for all wset dot product executors. + * Only cares about the underlying value type, not the concrete type of the + * attribute vector itself. + */ +template <typename BaseType> +class DotProductExecutorBase : public fef::FeatureExecutor { +public: + using AT = multivalue::WeightedValue<BaseType>; + using V = VectorBase<BaseType, BaseType, feature_t>; +private: + V _queryVector; + const typename V::HashMap::const_iterator _end; + virtual size_t getAttributeValues(uint32_t docid, const AT * & count) = 0; +public: + DotProductExecutorBase(V queryVector); + ~DotProductExecutorBase() override; + void execute(uint32_t docId) override; +}; + +template <typename A> +class DotProductExecutor final : public DotProductExecutorBase<typename A::BaseType> { +public: + using AT = typename DotProductExecutorBase<typename A::BaseType>::AT; + using V = typename DotProductExecutorBase<typename A::BaseType>::V; +protected: + const A * _attribute; +private: + size_t getAttributeValues(uint32_t docid, const AT * & count) override; +public: + DotProductExecutor(const A * attribute, V queryVector); + ~DotProductExecutor(); +}; + /** * Implements the executor for the dotproduct feature. */ template <typename Vector, typename Buffer> -class DotProductExecutor : public fef::FeatureExecutor { +class DotProductExecutorByCopy final : public fef::FeatureExecutor { private: const attribute::IAttributeVector * _attribute; - const Vector _queryVector; + Vector _queryVector; const typename Vector::HashMap::const_iterator _end; Buffer _buffer; public: - DotProductExecutor(const attribute::IAttributeVector * attribute, const Vector & queryVector); + DotProductExecutorByCopy(const attribute::IAttributeVector * attribute, Vector queryVector); + ~DotProductExecutorByCopy() override; void execute(uint32_t docId) override; }; @@ -143,7 +178,7 @@ private: virtual size_t getAttributeValues(uint32_t docid, const AT * & count) = 0; public: DotProductExecutorBase(const V & queryVector); - ~DotProductExecutorBase(); + ~DotProductExecutorBase() override; void execute(uint32_t docId) final override; }; diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp index 9d383e5a03a..eb73cef1f4c 100644 --- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp @@ -32,15 +32,15 @@ protected: IntegerVector _queryVector; public: - RawExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector); + RawExecutor(const IAttributeVector *attribute, IntegerVector queryVector); void execute(uint32_t docId) override; }; template <typename BaseType> -RawExecutor<BaseType>::RawExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector) : +RawExecutor<BaseType>::RawExecutor(const IAttributeVector *attribute, IntegerVector queryVector) : FeatureExecutor(), _attribute(attribute), - _queryVector(queryVector) + _queryVector(std::move(queryVector)) { _queryVector.syncMap(); } @@ -81,13 +81,13 @@ private: WeightedIntegerContent _buffer; public: - BufferedExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector); + BufferedExecutor(const IAttributeVector *attribute, IntegerVector queryVector); void execute(uint32_t docId) override; }; template <typename BaseType> -BufferedExecutor<BaseType>::BufferedExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector) : - RawExecutor<BaseType>(attribute, queryVector), +BufferedExecutor<BaseType>::BufferedExecutor(const IAttributeVector *attribute, IntegerVector queryVector) : + RawExecutor<BaseType>(attribute, std::move(queryVector)), _buffer() { } @@ -109,20 +109,17 @@ InternalMaxReduceProdJoinBlueprint::InternalMaxReduceProdJoinBlueprint() : { } -InternalMaxReduceProdJoinBlueprint::~InternalMaxReduceProdJoinBlueprint() -{ -} +InternalMaxReduceProdJoinBlueprint::~InternalMaxReduceProdJoinBlueprint() = default; void -InternalMaxReduceProdJoinBlueprint::visitDumpFeatures(const IIndexEnvironment &, - IDumpFeatureVisitor &) const +InternalMaxReduceProdJoinBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const { } Blueprint::UP InternalMaxReduceProdJoinBlueprint::createInstance() const { - return Blueprint::UP(new InternalMaxReduceProdJoinBlueprint()); + return std::make_unique<InternalMaxReduceProdJoinBlueprint>(); } ParameterDescriptions @@ -155,7 +152,7 @@ bool supportsGetRawValues(const A &attr) noexcept { template <typename BaseType> FeatureExecutor & -selectTypedExecutor(const IAttributeVector *attribute, const IntegerVector &vector, vespalib::Stash &stash) +selectTypedExecutor(const IAttributeVector *attribute, IntegerVector vector, vespalib::Stash &stash) { if (!attribute->isImported()) { using A = IntegerAttributeTemplate<BaseType>; @@ -166,22 +163,22 @@ selectTypedExecutor(const IAttributeVector *attribute, const IntegerVector &vect if (supportsGetRawValues(*iattr)) { const ExactA *exactA = dynamic_cast<const ExactA *>(iattr); if (exactA != nullptr) { - return stash.create<RawExecutor<BaseType>>(attribute, vector); + return stash.create<RawExecutor<BaseType>>(attribute, std::move(vector)); } } } - return stash.create<BufferedExecutor<BaseType>>(attribute, vector); + return stash.create<BufferedExecutor<BaseType>>(attribute, std::move(vector)); } FeatureExecutor & -selectExecutor(const IAttributeVector *attribute, const IntegerVector &vector, vespalib::Stash &stash) +selectExecutor(const IAttributeVector *attribute, IntegerVector vector, vespalib::Stash &stash) { if (attribute->getCollectionType() == CollectionType::ARRAY) { switch (attribute->getBasicType()) { case BasicType::INT32: - return selectTypedExecutor<int32_t>(attribute, vector, stash); + return selectTypedExecutor<int32_t>(attribute, std::move(vector), stash); case BasicType::INT64: - return selectTypedExecutor<int64_t>(attribute, vector, stash); + return selectTypedExecutor<int64_t>(attribute, std::move(vector), stash); default: break; } @@ -207,7 +204,7 @@ InternalMaxReduceProdJoinBlueprint::createExecutor(const IQueryEnvironment &env, IntegerVector vector; WeightedSetParser::parse(prop.get(), vector); if (!vector.getVector().empty()) { - return selectExecutor(attribute, vector, stash); + return selectExecutor(attribute, std::move(vector), stash); } } return stash.create<SingleZeroValueExecutor>(); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h index 619c1760797..345d69c29f6 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h @@ -17,7 +17,7 @@ namespace search { namespace fakedata { -/* +/** * General representation of a faked word, containing all features used * by any of the candidate posting list formats. */ diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp index 1f8ea9fb2ea..5c87bf88e9c 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp @@ -10,27 +10,23 @@ LOG_SETUP(".fakewordset"); namespace search::fakedata { +using FakeWordVector = FakeWordSet::FakeWordVector; using index::PostingListParams; using index::SchemaUtil; using index::schema::CollectionType; using index::schema::DataType; -static void -clearFakeWordVector(std::vector<FakeWord *> &v) -{ - for (unsigned int i = 0; i < v.size(); ++i) - delete v[i]; - v.clear(); -} - +namespace { -static void -applyDocIdBiasToVector(std::vector<FakeWord *> &v, uint32_t docIdBias) +void +applyDocIdBiasToVector(FakeWordVector& words, uint32_t docIdBias) { - for (unsigned int i = 0; i < v.size(); ++i) - v[i]->addDocIdBias(docIdBias); + for (auto& word : words) { + word->addDocIdBias(docIdBias); + } } +} FakeWordSet::FakeWordSet() : _words(NUM_WORDCLASSES), @@ -40,7 +36,6 @@ FakeWordSet::FakeWordSet() setupParams(false, false); } - FakeWordSet::FakeWordSet(bool hasElements, bool hasElementWeights) : _words(NUM_WORDCLASSES), @@ -50,12 +45,7 @@ FakeWordSet::FakeWordSet(bool hasElements, setupParams(hasElements, hasElementWeights); } - -FakeWordSet::~FakeWordSet() -{ - dropWords(); -} - +FakeWordSet::~FakeWordSet() = default; void FakeWordSet::setupParams(bool hasElements, @@ -83,7 +73,6 @@ FakeWordSet::setupParams(bool hasElements, } } - void FakeWordSet::setupWords(search::Rand48 &rnd, unsigned int numDocs, @@ -93,7 +82,6 @@ FakeWordSet::setupWords(search::Rand48 &rnd, std::string common = "common"; std::string medium = "medium"; std::string rare = "rare"; - FakeWord *fw; FastOS_Time tv; double before; double after; @@ -106,50 +94,42 @@ FakeWordSet::setupWords(search::Rand48 &rnd, std::ostringstream vi; vi << (i + 1); - fw = new FakeWord(numDocs, commonDocFreq, commonDocFreq / 2, - common + vi.str(), rnd, - _fieldsParams[packedIndex], - packedIndex); - _words[COMMON_WORD].push_back(fw); - fw = new FakeWord(numDocs, 1000, 500, - medium + vi.str(), rnd, - _fieldsParams[packedIndex], - packedIndex); - _words[MEDIUM_WORD].push_back(fw); - fw = new FakeWord(numDocs, 10, 5, - rare + vi.str(), rnd, - _fieldsParams[packedIndex], - packedIndex); - _words[RARE_WORD].push_back(fw); + _words[COMMON_WORD].push_back(std::make_unique<FakeWord>(numDocs, commonDocFreq, commonDocFreq / 2, + common + vi.str(), rnd, + _fieldsParams[packedIndex], + packedIndex)); + + _words[MEDIUM_WORD].push_back(std::make_unique<FakeWord>(numDocs, 1000, 500, + medium + vi.str(), rnd, + _fieldsParams[packedIndex], + packedIndex)); + + _words[RARE_WORD].push_back(std::make_unique<FakeWord>(numDocs, 10, 5, + rare + vi.str(), rnd, + _fieldsParams[packedIndex], + packedIndex)); } tv.SetNow(); after = tv.Secs(); LOG(info, "leave setupWords, elapsed %10.6f s", after - before); } - -void -FakeWordSet::dropWords() -{ - for (unsigned int i = 0; i < _words.size(); ++i) - clearFakeWordVector(_words[i]); -} - - int -FakeWordSet::getNumWords() +FakeWordSet::getNumWords() const { int ret = 0; - for (unsigned int i = 0; i < _words.size(); ++i) - ret += _words[i].size(); + for (const auto& words : _words) { + ret += words.size(); + } return ret; } void FakeWordSet::addDocIdBias(uint32_t docIdBias) { - for (unsigned int i = 0; i < _words.size(); ++i) - applyDocIdBiasToVector(_words[i], docIdBias); + for (auto& words : _words) { + applyDocIdBiasToVector(words, docIdBias); + } } } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h index 754a592820c..0b7ee4db6fe 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h @@ -5,25 +5,21 @@ #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -namespace search -{ -class Rand48; -} - -namespace search -{ +namespace search { class Rand48; } -namespace fakedata -{ +namespace search::fakedata { class FakeWord; -class FakeWordSet -{ +/** + * Contains lists of fake words for 3 word classes categorized based on number of occurrences. + */ +class FakeWordSet { public: - typedef bitcompression::PosOccFieldsParams PosOccFieldsParams; - typedef bitcompression::PosOccFieldParams PosOccFieldParams; - typedef index::Schema Schema; + using PosOccFieldsParams = bitcompression::PosOccFieldsParams; + using Schema = index::Schema; + using FakeWordPtr = std::unique_ptr<FakeWord>; + using FakeWordVector = std::vector<FakeWordPtr>; enum { COMMON_WORD, @@ -31,10 +27,13 @@ public: RARE_WORD, NUM_WORDCLASSES, }; - std::vector<std::vector<FakeWord *> > _words; + +private: + std::vector<FakeWordVector> _words; Schema _schema; std::vector<PosOccFieldsParams> _fieldsParams; +public: FakeWordSet(); FakeWordSet(bool hasElements, @@ -42,51 +41,36 @@ public: ~FakeWordSet(); - void - setupParams(bool hasElements, - bool hasElementWeights); + void setupParams(bool hasElements, + bool hasElementWeights); - void - setupWords(search::Rand48 &rnd, - unsigned int numDocs, - unsigned int commonDocFreq, - unsigned int numWordsPerWordClass); + void setupWords(search::Rand48 &rnd, + unsigned int numDocs, + unsigned int commonDocFreq, + unsigned int numWordsPerWordClass); - void - dropWords(); + const std::vector<FakeWordVector>& words() const { return _words; } - int - getNumWords(); + int getNumWords() const; - const PosOccFieldsParams & - getFieldsParams() const - { + const PosOccFieldsParams& getFieldsParams() const { return _fieldsParams.back(); } - uint32_t - getPackedIndex() const - { + uint32_t getPackedIndex() const { return _fieldsParams.size() - 1; } - const std::vector<PosOccFieldsParams> & - getAllFieldsParams() const - { + const std::vector<PosOccFieldsParams>& getAllFieldsParams() const { return _fieldsParams; } - const Schema & - getSchema() const - { + const Schema& getSchema() const { return _schema; } - void - addDocIdBias(uint32_t docIdBias); + void addDocIdBias(uint32_t docIdBias); }; -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.cpp index f10af286dd7..76bfeac759f 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.cpp @@ -19,12 +19,9 @@ FPFactory::setup(const FakeWordSet &fws) { std::vector<const FakeWord *> v; - for (uint32_t wc = 0; wc < fws._words.size(); ++wc) { - std::vector<FakeWord *>::const_iterator fwi(fws._words[wc].begin()); - std::vector<FakeWord *>::const_iterator fwe(fws._words[wc].end()); - while (fwi != fwe) { - v.push_back(*fwi); - ++fwi; + for (const auto& words : fws.words()) { + for (const auto& word : words) { + v.push_back(word.get()); } } setup(v); diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java new file mode 100644 index 00000000000..b5822ff825e --- /dev/null +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -0,0 +1,78 @@ +package ai.vespa.hosted.plugin; + +import ai.vespa.hosted.api.ControllerHttpClient; +import com.yahoo.config.provision.ApplicationId; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; + +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Base class for hosted Vespa plugin mojos. + * + * @author jonmv + */ +public abstract class AbstractVespaMojo extends AbstractMojo { + + @Parameter(defaultValue = "${project}", readonly = true) + protected MavenProject project; + + @Parameter(property = "endpoint", defaultValue = "https://api.vespa.corp.yahoo.com:4443") // TODO jvenstad: Change default + protected String endpointUri; + + @Parameter(property = "tenant") + protected String tenant; + + @Parameter(property = "application") + protected String application; + + @Parameter(property = "instance") + protected String instance; + + @Parameter(property = "privateKeyFile", required = true) + protected String privateKeyFile; + + @Parameter(property = "certificateFile") + protected String certificateFile; + + // Fields set up as part of setup(). + protected ApplicationId id; + protected ControllerHttpClient controller; + + @Override + public final void execute() { + setup(); + doExecute(); + } + + /** Override this in subclasses, instead of {@link #execute()}. */ + protected abstract void doExecute(); + + protected void setup() { + tenant = firstNonBlank(tenant, project.getProperties().getProperty("tenant")); + application = firstNonBlank(application, project.getProperties().getProperty("application")); + instance = firstNonBlank(instance, project.getProperties().getProperty("instance"), "default"); + id = ApplicationId.from(tenant, application, instance); + + controller = certificateFile == null + ? ControllerHttpClient.withSignatureKey(URI.create(endpointUri), Paths.get(privateKeyFile), id) + : ControllerHttpClient.withKeyAndCertificate(URI.create(endpointUri), Paths.get(privateKeyFile), Paths.get(certificateFile)); + } + + protected String projectPathOf(String first, String... rest) { + return project.getBasedir().toPath().resolve(Path.of(first, rest)).toString(); + } + + /** Returns the first of the given strings which is non-null and non-blank, or throws IllegalArgumentException. */ + protected static String firstNonBlank(String... values) { + for (String value : values) + if (value != null && ! value.isBlank()) + return value; + + throw new IllegalArgumentException("No valid value given"); + } + +} diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java new file mode 100644 index 00000000000..24e2c42d756 --- /dev/null +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java @@ -0,0 +1,18 @@ +package ai.vespa.hosted.plugin; + +import org.apache.maven.plugins.annotations.Mojo; + +/** + * Finds the Vespa version to compile against, for a hosted Vespa application. + * + * @author jonmv + */ +@Mojo(name = "compileVersion") +public class CompileVersionMojo extends AbstractVespaMojo { + + @Override + protected void doExecute() { + System.out.println(controller.compileVersion(id)); + } + +} diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java new file mode 100644 index 00000000000..9a2c65ef86e --- /dev/null +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java @@ -0,0 +1,58 @@ +package ai.vespa.hosted.plugin; + +import ai.vespa.hosted.api.Deployment; +import com.yahoo.config.provision.zone.ZoneId; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + +import java.nio.file.Paths; + +/** + * Deploys a Vespa application package to the hosted Vespa API. + * + * @author jonmv + */ +@Mojo(name = "deploy") +public class DeployMojo extends AbstractVespaMojo { + + @Parameter(property = "applicationZip") + private String applicationZip; + + @Parameter(property = "vespaVersion") + private String vespaVersion; + + @Parameter(property = "ignoreValidationErrors") + private String ignoreValidationErrors; + + @Parameter(property = "environment") + private String environment; + + @Parameter(property = "region") + private String region; + + @Parameter(property = "repository") + private String repository; + + @Parameter(property = "branch") + private String branch; + + @Parameter(property = "commit") + private String commit; + + @Parameter(property = "build") + private Long build; + + @Override + protected void doExecute() { + Deployment deployment = build == null + ? Deployment.ofPackage(Paths.get(firstNonBlank(applicationZip, projectPathOf("target", "application.zip")))) + : Deployment.ofReference(repository, branch, commit, build); + if ("true".equalsIgnoreCase(ignoreValidationErrors)) deployment = deployment.ignoringValidationErrors(); + if (vespaVersion != null) deployment = deployment.atVersion(vespaVersion); + + ZoneId zone = environment == null || region == null ? controller.devZone() : ZoneId.from(environment, region); + + System.out.println(controller.deploy(deployment, id, zone).json()); + } + +} diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java index 6f39c29963c..eda9ea9b799 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java @@ -1,17 +1,12 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; -import ai.vespa.hosted.api.ControllerHttpClient; import ai.vespa.hosted.api.Submission; -import com.yahoo.config.provision.ApplicationId; -import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; -import org.apache.maven.project.MavenProject; -import java.net.URI; -import java.nio.file.Path; import java.nio.file.Paths; +import java.util.OptionalLong; /** * Submits a Vespa application package and corresponding test jars to the hosted Vespa API. @@ -19,22 +14,7 @@ import java.nio.file.Paths; * @author jonmv */ @Mojo(name = "submit") -public class SubmitMojo extends AbstractMojo { - - @Parameter(defaultValue = "${project}", readonly = true) - private MavenProject project; - - @Parameter(property = "endpoint", defaultValue = "https://api.vespa.corp.yahoo.com:4443") // TODO jvenstad: Change default - private String endpointUri; - - @Parameter(property = "tenant") - private String tenant; - - @Parameter(property = "application") - private String application; - - @Parameter(property = "instance") - private String instance; +public class SubmitMojo extends AbstractVespaMojo { @Parameter(property = "applicationZip") private String applicationZip; @@ -42,12 +22,6 @@ public class SubmitMojo extends AbstractMojo { @Parameter(property = "applicationTestZip") private String applicationTestZip; - @Parameter(property = "privateKeyFile", required = true) - private String privateKeyFile; - - @Parameter(property = "certificateFile") - private String certificateFile; - @Parameter(property = "authorEmail", required = true) private String authorEmail; @@ -60,39 +34,19 @@ public class SubmitMojo extends AbstractMojo { @Parameter(property = "commit", defaultValue = "unknown") private String commit; - @Override - public void execute() { - setup(); - ApplicationId id = ApplicationId.from(tenant, application, instance); - ControllerHttpClient controller = certificateFile == null - ? ControllerHttpClient.withSignatureKey(URI.create(endpointUri), Paths.get(privateKeyFile), id) - : ControllerHttpClient.withKeyAndCertificate(URI.create(endpointUri), Paths.get(privateKeyFile), Paths.get(certificateFile)); - - Submission submission = new Submission(repository, branch, commit, authorEmail, - Paths.get(applicationZip), Paths.get(applicationTestZip)); - - System.out.println(controller.submit(submission, id.tenant(), id.application())); - } + @Parameter(property = "projectId") + private Long projectId; - private void setup() { - tenant = firstNonBlank(tenant, project.getProperties().getProperty("tenant")); - application = firstNonBlank(application, project.getProperties().getProperty("application")); - instance = firstNonBlank(instance, project.getProperties().getProperty("instance")); + @Override + public void doExecute() { applicationZip = firstNonBlank(applicationZip, projectPathOf("target", "application.zip")); applicationTestZip = firstNonBlank(applicationTestZip, projectPathOf("target", "application-test.zip")); - } - - private String projectPathOf(String first, String... rest) { - return project.getBasedir().toPath().resolve(Path.of(first, rest)).toString(); - } - - /** Returns the first of the given strings which is non-null and non-blank, or throws IllegalArgumentException. */ - private static String firstNonBlank(String... values) { - for (String value : values) - if (value != null && ! value.isBlank()) - return value; + Submission submission = new Submission(repository, branch, commit, authorEmail, + Paths.get(applicationZip), + Paths.get(applicationTestZip), + projectId == null ? OptionalLong.empty() : OptionalLong.of(projectId)); - throw new IllegalArgumentException("No valid value given"); + System.out.println(controller.submit(submission, id.tenant(), id.application())); } } |