summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorValerij Fredriksen <freva@users.noreply.github.com>2019-07-05 12:38:57 +0200
committerGitHub <noreply@github.com>2019-07-05 12:38:57 +0200
commitd3052fde0fe524888a51a41f319be92202e2ae43 (patch)
treeea2e27cb282e44bac5dc2ce115ce7d6fe8cde7d9
parent10e4c09fa1db4ad4c10b12d216bba6cc33593dc7 (diff)
parent36d91d1cf1c30b6e0e8f6aad4c449e2d75fcb3aa (diff)
Merge pull request #9959 from vespa-engine/freva/remove-metrics
Node-Admin: Remove node metrics related code
Diffstat
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java18
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java1
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java12
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java132
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java1
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java17
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java3
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java222
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java3
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java1
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java2
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java4
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java2
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java101
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/package-info.java5
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java165
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java99
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java10
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java88
-rw-r--r--node-admin/src/test/resources/docker.stats.json376
-rw-r--r--node-admin/src/test/resources/expected.container.system.metrics.0.txt78
-rw-r--r--node-admin/src/test/resources/expected.container.system.metrics.1.txt82
24 files changed, 62 insertions, 1376 deletions
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java
index 46a0f9b9b10..590ef207e3f 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java
@@ -1,22 +1,15 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.dockerapi.metrics;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.TreeMap;
import java.util.stream.Collectors;
/**
* @author freva
*/
public class DimensionMetrics {
- private static final ObjectMapper objectMapper = new ObjectMapper();
- private static final Map<String, Object> routing = Map.of("yamas", Map.of("namespaces", List.of("Vespa")));
private final String application;
private final Dimensions dimensions;
@@ -30,17 +23,6 @@ public class DimensionMetrics {
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
- public String toSecretAgentReport() throws JsonProcessingException {
- Map<String, Object> report = new TreeMap<>();
- report.put("application", application);
- report.put("dimensions", new TreeMap<>(dimensions.asMap()));
- report.put("metrics", new TreeMap<>(metrics));
- report.put("routing", routing);
- report.put("timestamp", System.currentTimeMillis() / 1000);
-
- return objectMapper.writeValueAsString(report);
- }
-
public String getApplication() {
return application;
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
index 1811fc0c8f0..d515f0d0353 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.hosted.node.admin.component;
import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzService;
import java.net.URI;
import java.util.Collections;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index d8d6b4781c8..f67c243f34d 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -42,8 +42,8 @@ public class DockerOperationsImpl implements DockerOperations {
private static final String MANAGER_NAME = "node-admin";
- private static final String IPV6_NPT_PREFIX = "fd00::";
- private static final String IPV4_NPT_PREFIX = "172.17.0.0";
+ private static final InetAddress IPV6_NPT_PREFIX = InetAddresses.forString("fd00::");
+ private static final InetAddress IPV4_NPT_PREFIX = InetAddresses.forString("172.17.0.0");
private final Docker docker;
private final ProcessExecuter processExecuter;
@@ -96,16 +96,12 @@ public class DockerOperationsImpl implements DockerOperations {
command.withNetworkMode(networking.getDockerNetworkMode());
if (networking == DockerNetworking.NPT) {
- InetAddress ipV6Prefix = InetAddresses.forString(IPV6_NPT_PREFIX);
- InetAddress ipV6Local = IPAddresses.prefixTranslate(ipV6Address, ipV6Prefix, 8);
+ InetAddress ipV6Local = IPAddresses.prefixTranslate(ipV6Address, IPV6_NPT_PREFIX, 8);
command.withIpAddress(ipV6Local);
// IPv4 - Only present for some containers
Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().hostname())
- .map(ipV4Address -> {
- InetAddress ipV4Prefix = InetAddresses.forString(IPV4_NPT_PREFIX);
- return IPAddresses.prefixTranslate(ipV4Address, ipV4Prefix, 2);
- });
+ .map(ipV4Address -> IPAddresses.prefixTranslate(ipV4Address, IPV4_NPT_PREFIX, 2));
ipV4Local.ifPresent(command::withIpAddress);
addEtcHosts(containerData, context.node().hostname(), ipV4Local, ipV6Local);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 26e4dcda88e..167ca15bdbf 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -3,17 +3,14 @@ package com.yahoo.vespa.hosted.node.admin.maintenance;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
-import com.yahoo.config.provision.NodeType;
import com.yahoo.log.LogLevel;
import com.yahoo.vespa.hosted.dockerapi.Container;
import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
-import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -22,11 +19,8 @@ import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@@ -35,7 +29,6 @@ import java.util.regex.Pattern;
import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches;
import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.olderThan;
-import static com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig.nodeTypeToRole;
import static com.yahoo.yolean.Exceptions.uncheck;
/**
@@ -47,7 +40,6 @@ public class StorageMaintainer {
.ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC);
private final Terminal terminal;
- private final DockerOperations dockerOperations;
private final CoredumpHandler coredumpHandler;
private final Path archiveContainerStoragePath;
@@ -57,134 +49,12 @@ public class StorageMaintainer {
.expireAfterWrite(5, TimeUnit.MINUTES)
.build();
- public StorageMaintainer(Terminal terminal, DockerOperations dockerOperations, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) {
+ public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) {
this.terminal = terminal;
- this.dockerOperations = dockerOperations;
this.coredumpHandler = coredumpHandler;
this.archiveContainerStoragePath = archiveContainerStoragePath;
}
- public void writeMetricsConfig(NodeAgentContext context) {
- List<SecretAgentCheckConfig> configs = new ArrayList<>();
- Map<String, Object> tags = generateTags(context);
-
- // host-life
- Path hostLifeCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life");
- configs.add(new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath).withTags(tags));
-
- // coredumps (except for the done coredumps which is handled by the host)
- Path coredumpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps");
- configs.add(new SecretAgentCheckConfig("system-coredumps-processing", 300, coredumpCheckPath,
- "--application", "system-coredumps-processing",
- "--lastmin", "129600",
- "--crit", "1",
- "--coredir", context.pathInNodeUnderVespaHome("var/crash/processing").toString())
- .withTags(tags));
-
- // athenz certificate check
- Path athenzCertExpiryCheckPath = context.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs");
- configs.add(new SecretAgentCheckConfig("athenz-certificate-expiry", 60, athenzCertExpiryCheckPath,
- "--threshold", "20")
- .withRunAsUser("root")
- .withTags(tags));
-
- if (context.nodeType() != NodeType.config) {
- // vespa-health
- Path vespaHealthCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health");
- configs.add(new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all")
- .withRunAsUser(context.vespaUser())
- .withTags(tags));
-
- // vespa
- Path vespaCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa");
- SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all");
- vespaSchedule.withRunAsUser(context.vespaUser());
- if (isConfigserverLike(context.nodeType())) {
- Map<String, Object> tagsWithoutNameSpace = new LinkedHashMap<>(tags);
- tagsWithoutNameSpace.remove("namespace");
- vespaSchedule.withTags(tagsWithoutNameSpace);
- }
- configs.add(vespaSchedule);
- }
-
- if (context.nodeType() == NodeType.config || context.nodeType() == NodeType.controller) {
-
- // configserver/controller
- Path configServerNewCheckPath = Paths.get("/usr/bin/curl");
- configs.add(new SecretAgentCheckConfig(nodeTypeToRole(context.nodeType()), 60, configServerNewCheckPath,
- "-s", "localhost:19071/yamas-metrics")
- .withTags(tags));
-
- //zkbackupage
- Path zkbackupCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
- configs.add(new SecretAgentCheckConfig("zkbackupage", 300, zkbackupCheckPath,
- "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(),
- "-m", "150",
- "-a", "config-zkbackupage")
- .withTags(tags));
-
- String appName = nodeTypeToRole(context.nodeType()) + "-logd";
- Path logdCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/convert-state-metrics-2-yamas.py");
- configs.add(new SecretAgentCheckConfig(appName, 60, logdCheckPath,
- appName, "http://localhost:19089/state/v1/metrics")
- .withTags(tags));
- }
-
- if (context.nodeType() == NodeType.proxy) {
- //routing-configage
- Path routingAgeCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
- configs.add(new SecretAgentCheckConfig("routing-configage", 60, routingAgeCheckPath,
- "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf.tmp").toString(),
- "-m", "1",
- "-a", "routing-configage",
- "--ignore_file_not_found")
- .withTags(tags));
-
- //ssl-check
- Path sslCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status");
- configs.add(new SecretAgentCheckConfig("ssl-status", 300, sslCheckPath,
- "-e", "localhost",
- "-p", "4443",
- "-t", "30")
- .withTags(tags));
- }
-
- // Write config and restart yamas-agent
- Path yamasAgentFolder = context.pathOnHostFromPathInNode("/etc/yamas-agent");
- configs.forEach(s -> uncheck(() -> s.writeTo(yamasAgentFolder)));
- dockerOperations.executeCommandInContainerAsRoot(context, "service", "yamas-agent", "restart");
- }
-
- private Map<String, Object> generateTags(NodeAgentContext context) {
- Map<String, String> tags = new LinkedHashMap<>();
- tags.put("namespace", "Vespa");
- tags.put("role", nodeTypeToRole(context.node().type()));
- tags.put("zone", context.zone().getId().value());
- context.node().currentVespaVersion().ifPresent(version -> tags.put("vespaVersion", version.toFullString()));
-
- if (! isConfigserverLike(context.nodeType())) {
- tags.put("state", context.node().state().toString());
- context.node().parentHostname().ifPresent(parent -> tags.put("parentHostname", parent));
- context.node().owner().ifPresent(owner -> {
- tags.put("tenantName", owner.tenant());
- tags.put("app", owner.application() + "." + owner.instance());
- tags.put("applicationName", owner.application());
- tags.put("instanceName", owner.instance());
- tags.put("applicationId", owner.tenant() + "." + owner.application() + "." + owner.instance());
- });
- context.node().membership().ifPresent(membership -> {
- tags.put("clustertype", membership.clusterType());
- tags.put("clusterid", membership.clusterId());
- });
- }
-
- return Collections.unmodifiableMap(tags);
- }
-
- private boolean isConfigserverLike(NodeType nodeType) {
- return nodeType == NodeType.config || nodeType == NodeType.controller;
- }
-
public Optional<Long> getDiskUsageFor(NodeAgentContext context) {
try {
Path path = context.pathOnHostFromPathInNode("/");
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
index 550d6e7021e..ce7a99fd841 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
@@ -9,7 +9,6 @@ import com.yahoo.security.Pkcs10Csr;
import com.yahoo.security.SslContextBuilder;
import com.yahoo.security.X509CertificateUtils;
import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzService;
import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient;
import com.yahoo.vespa.athenz.client.zts.InstanceIdentity;
import com.yahoo.vespa.athenz.client.zts.ZtsClient;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
index cb10eac9e6c..5d2639d0a77 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
@@ -149,7 +149,7 @@ public class NodeAdminImpl implements NodeAdmin {
@Override
public Duration subsystemFreezeDuration() {
if (startOfFreezeConvergence == null) {
- return Duration.ofSeconds(0);
+ return Duration.ZERO;
} else {
return Duration.between(startOfFreezeConvergence, clock.instant());
}
@@ -172,7 +172,7 @@ public class NodeAdminImpl implements NodeAdmin {
@Override
public void stop() {
// Stop all node-agents in parallel, will block until the last NodeAgent is stopped
- nodeAgentWithSchedulerByHostname.values().parallelStream().forEach(NodeAgent::stopForRemoval);
+ nodeAgentWithSchedulerByHostname.values().parallelStream().forEach(NodeAgentWithScheduler::stopForRemoval);
}
// Set-difference. Returns minuend minus subtrahend.
@@ -182,7 +182,7 @@ public class NodeAdminImpl implements NodeAdmin {
return result;
}
- static class NodeAgentWithScheduler implements NodeAgent, NodeAgentScheduler {
+ static class NodeAgentWithScheduler implements NodeAgentScheduler {
private final NodeAgent nodeAgent;
private final NodeAgentScheduler nodeAgentScheduler;
@@ -191,14 +191,15 @@ public class NodeAdminImpl implements NodeAdmin {
this.nodeAgentScheduler = nodeAgentScheduler;
}
- @Override public void start() { nodeAgent.start(); }
- @Override public void stopForHostSuspension() { nodeAgent.stopForHostSuspension(); }
- @Override public void stopForRemoval() { nodeAgent.stopForRemoval(); }
- @Override public void updateContainerNodeMetrics() { nodeAgent.updateContainerNodeMetrics(); }
- @Override public int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); }
+ void start() { nodeAgent.start(currentContext()); }
+ void stopForHostSuspension() { nodeAgent.stopForHostSuspension(currentContext()); }
+ void stopForRemoval() { nodeAgent.stopForRemoval(currentContext()); }
+ void updateContainerNodeMetrics() { nodeAgent.updateContainerNodeMetrics(currentContext()); }
+ int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); }
@Override public void scheduleTickWith(NodeAgentContext context, Instant at) { nodeAgentScheduler.scheduleTickWith(context, at); }
@Override public boolean setFrozen(boolean frozen, Duration timeout) { return nodeAgentScheduler.setFrozen(frozen, timeout); }
+ @Override public NodeAgentContext currentContext() { return nodeAgentScheduler.currentContext(); }
}
@FunctionalInterface
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java
index de5ee1b69a4..f537884e708 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java
@@ -13,24 +13,24 @@ public interface NodeAgent {
* Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously
* striving to make the current state equal to the wanted state.
*/
- void start();
+ void start(NodeAgentContext context);
/**
* Stop the node in anticipation of host suspension, e.g. reboot or docker upgrade.
*/
- void stopForHostSuspension();
+ void stopForHostSuspension(NodeAgentContext context);
/**
* Signals to the agent that the node is at the end of its lifecycle and no longer needs a managing agent.
* Cleans up any resources the agent owns, such as threads, connections etc. Cleanup is synchronous; when this
* method returns, no more actions will be taken by the agent.
*/
- void stopForRemoval();
+ void stopForRemoval(NodeAgentContext context);
/**
* Updates metric receiver with the latest node-agent stats
*/
- void updateContainerNodeMetrics();
+ default void updateContainerNodeMetrics(NodeAgentContext context) {}
/**
* Returns and resets number of unhandled exceptions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java
index 1fc730a3cb0..65611886f9c 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java
@@ -13,9 +13,6 @@ public interface NodeAgentContextSupplier {
*/
NodeAgentContext nextContext() throws InterruptedException;
- /** @return the last context returned by {@link #nextContext()} or a default value */
- NodeAgentContext currentContext();
-
/** Interrupts the thread(s) currently waiting in {@link #nextContext()} */
void interrupt();
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 90eda96d445..77c08133e82 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.admin.nodeagent;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.zone.ZoneApi;
@@ -12,13 +11,8 @@ import com.yahoo.vespa.flags.FlagSource;
import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.hosted.dockerapi.Container;
import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
-import com.yahoo.vespa.hosted.dockerapi.ContainerStats;
import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException;
import com.yahoo.vespa.hosted.dockerapi.exception.DockerException;
-import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException;
-import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics;
-import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions;
-import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
@@ -31,11 +25,8 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer;
import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer;
import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
-import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig;
import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -68,16 +59,16 @@ public class NodeAgentImpl implements NodeAgent {
private final Optional<CredentialsMaintainer> credentialsMaintainer;
private final Optional<AclMaintainer> aclMaintainer;
private final Optional<HealthChecker> healthChecker;
-
private final DoubleFlag containerCpuCap;
+ private Thread loopThread;
+ private ContainerState containerState = UNKNOWN;
+ private NodeSpec lastNode;
+
private int numberOfUnhandledException = 0;
private long currentRebootGeneration = 0;
private Optional<Long> currentRestartGeneration = Optional.empty();
- private final Thread loopThread;
-
-
/**
* ABSENT means container is definitely absent - A container that was absent will not suddenly appear without
* NodeAgent explicitly starting it.
@@ -92,10 +83,6 @@ public class NodeAgentImpl implements NodeAgent {
UNKNOWN
}
- private ContainerState containerState = UNKNOWN;
-
- private NodeSpec lastNode = null;
- private CpuUsageReporter lastCpuMetric = new CpuUsageReporter();
// Created in NodeAdminImpl
public NodeAgentImpl(
@@ -116,11 +103,15 @@ public class NodeAgentImpl implements NodeAgent {
this.credentialsMaintainer = credentialsMaintainer;
this.aclMaintainer = aclMaintainer;
this.healthChecker = healthChecker;
+ this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource);
+ }
- this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource)
- .with(FetchVector.Dimension.HOSTNAME, contextSupplier.currentContext().node().hostname());
+ @Override
+ public void start(NodeAgentContext initialContext) {
+ if (loopThread != null)
+ throw new IllegalStateException("Can not re-start a node agent.");
- this.loopThread = new Thread(() -> {
+ loopThread = new Thread(() -> {
while (!terminated.get()) {
try {
NodeAgentContext context = contextSupplier.nextContext();
@@ -128,19 +119,15 @@ public class NodeAgentImpl implements NodeAgent {
} catch (InterruptedException ignored) { }
}
});
- this.loopThread.setName("tick-" + contextSupplier.currentContext().hostname());
- }
-
- @Override
- public void start() {
+ loopThread.setName("tick-" + initialContext.hostname());
loopThread.start();
}
@Override
- public void stopForRemoval() {
- if (!terminated.compareAndSet(false, true)) {
- throw new RuntimeException("Can not re-stop a node agent.");
- }
+ public void stopForRemoval(NodeAgentContext context) {
+ if (!terminated.compareAndSet(false, true))
+ throw new IllegalStateException("Can not re-stop a node agent.");
+
contextSupplier.interrupt();
do {
@@ -149,7 +136,7 @@ public class NodeAgentImpl implements NodeAgent {
} catch (InterruptedException ignored) { }
} while (loopThread.isAlive());
- contextSupplier.currentContext().log(logger, "Stopped");
+ context.log(logger, "Stopped");
}
void startServicesIfNeeded(NodeAgentContext context) {
@@ -209,7 +196,6 @@ public class NodeAgentImpl implements NodeAgent {
ContainerData containerData = createContainerData(context);
dockerOperations.createContainer(context, containerData, getContainerResources(context));
dockerOperations.startContainer(context);
- lastCpuMetric = new CpuUsageReporter();
hasStartedServices = true; // Automatically started with the container
hasResumedNode = false;
@@ -255,8 +241,7 @@ public class NodeAgentImpl implements NodeAgent {
}
}
- private void stopServices() {
- NodeAgentContext context = contextSupplier.currentContext();
+ private void stopServices(NodeAgentContext context) {
context.log(logger, "Stopping services");
if (containerState == ABSENT) return;
try {
@@ -268,13 +253,11 @@ public class NodeAgentImpl implements NodeAgent {
}
@Override
- public void stopForHostSuspension() {
- NodeAgentContext context = contextSupplier.currentContext();
+ public void stopForHostSuspension(NodeAgentContext context) {
getContainer(context).ifPresent(container -> removeContainer(context, container, "suspending host", true));
}
- public void suspend() {
- NodeAgentContext context = contextSupplier.currentContext();
+ public void suspend(NodeAgentContext context) {
context.log(logger, "Suspending services on node");
if (containerState == ABSENT) return;
try {
@@ -331,9 +314,9 @@ public class NodeAgentImpl implements NodeAgent {
try {
if (context.node().state() != NodeState.dirty) {
- suspend();
+ suspend(context);
}
- stopServices();
+ stopServices(context);
} catch (Exception e) {
context.log(logger, LogLevel.WARNING, "Failed stopping services, ignoring", e);
}
@@ -365,6 +348,7 @@ public class NodeAgentImpl implements NodeAgent {
.map(NodeOwner::asApplicationId)
.map(appId -> containerCpuCap.with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm()))
.orElse(containerCpuCap)
+ .with(FetchVector.Dimension.HOSTNAME, context.node().hostname())
.value() * context.node().vcpus();
return ContainerResources.from(cpuCap, context.node().vcpus(), context.node().memoryGb());
@@ -415,12 +399,6 @@ public class NodeAgentImpl implements NodeAgent {
currentRestartGeneration.map(current -> current < node.currentRestartGeneration().get()).orElse(false))
currentRestartGeneration = node.currentRestartGeneration();
- // Every time the node spec changes, we should clear the metrics for this container as the dimensions
- // will change and we will be reporting duplicate metrics.
- if (container.map(c -> c.state.isRunning()).orElse(false)) {
- storageMaintainer.writeMetricsConfig(context);
- }
-
lastNode = node;
}
@@ -513,100 +491,6 @@ public class NodeAgentImpl implements NodeAgent {
}
}
- @SuppressWarnings("unchecked")
- public void updateContainerNodeMetrics() {
- if (containerState != UNKNOWN) return;
- final NodeAgentContext context = contextSupplier.currentContext();
- final NodeSpec node = context.node();
-
- Optional<ContainerStats> containerStats = dockerOperations.getContainerStats(context);
- if (!containerStats.isPresent()) return;
-
- Dimensions.Builder dimensionsBuilder = new Dimensions.Builder()
- .add("host", context.hostname().value())
- .add("role", SecretAgentCheckConfig.nodeTypeToRole(context.nodeType()))
- .add("state", node.state().toString());
- node.parentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent));
- node.allowedToBeDown().ifPresent(allowed ->
- dimensionsBuilder.add("orchestratorState", allowed ? "ALLOWED_TO_BE_DOWN" : "NO_REMARKS"));
- Dimensions dimensions = dimensionsBuilder.build();
-
- ContainerStats stats = containerStats.get();
- final String APP = Metrics.APPLICATION_NODE;
- final int totalNumCpuCores = stats.getCpuStats().getOnlineCpus();
- final long memoryTotalBytes = stats.getMemoryStats().getLimit();
- final long memoryTotalBytesUsage = stats.getMemoryStats().getUsage();
- final long memoryTotalBytesCache = stats.getMemoryStats().getCache();
- final long diskTotalBytes = (long) (node.diskGb() * BYTES_IN_GB);
- final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context);
-
- lastCpuMetric.updateCpuDeltas(stats.getCpuStats());
-
- // Ratio of CPU cores allocated to this container to total number of CPU cores on this host
- final double allocatedCpuRatio = node.vcpus() / totalNumCpuCores;
- double cpuUsageRatioOfAllocated = lastCpuMetric.getCpuUsageRatio() / allocatedCpuRatio;
- double cpuKernelUsageRatioOfAllocated = lastCpuMetric.getCpuKernelUsageRatio() / allocatedCpuRatio;
- double cpuThrottledTimeRate = lastCpuMetric.getThrottledTimeRate();
- double cpuThrottledCpuTimeRate = lastCpuMetric.getThrottledCpuTimeRate();
-
- long memoryTotalBytesUsed = memoryTotalBytesUsage - memoryTotalBytesCache;
- double memoryUsageRatio = (double) memoryTotalBytesUsed / memoryTotalBytes;
- double memoryTotalUsageRatio = (double) memoryTotalBytesUsage / memoryTotalBytes;
- Optional<Double> diskUsageRatio = diskTotalBytesUsed.map(used -> (double) used / diskTotalBytes);
-
- List<DimensionMetrics> metrics = new ArrayList<>();
- DimensionMetrics.Builder systemMetricsBuilder = new DimensionMetrics.Builder(APP, dimensions)
- .withMetric("mem.limit", memoryTotalBytes)
- .withMetric("mem.used", memoryTotalBytesUsed)
- .withMetric("mem.util", 100 * memoryUsageRatio)
- .withMetric("mem_total.used", memoryTotalBytesUsage)
- .withMetric("mem_total.util", 100 * memoryTotalUsageRatio)
- .withMetric("cpu.util", 100 * cpuUsageRatioOfAllocated)
- .withMetric("cpu.sys.util", 100 * cpuKernelUsageRatioOfAllocated)
- .withMetric("cpu.throttled_time.rate", cpuThrottledTimeRate)
- .withMetric("cpu.throttled_cpu_time.rate", cpuThrottledCpuTimeRate)
- .withMetric("cpu.vcpus", node.vcpus())
- .withMetric("disk.limit", diskTotalBytes);
-
- diskTotalBytesUsed.ifPresent(diskUsed -> systemMetricsBuilder.withMetric("disk.used", diskUsed));
- diskUsageRatio.ifPresent(diskRatio -> systemMetricsBuilder.withMetric("disk.util", 100 * diskRatio));
- metrics.add(systemMetricsBuilder.build());
-
- stats.getNetworks().forEach((interfaceName, interfaceStats) -> {
- Dimensions netDims = dimensionsBuilder.add("interface", interfaceName).build();
- DimensionMetrics networkMetrics = new DimensionMetrics.Builder(APP, netDims)
- .withMetric("net.in.bytes", interfaceStats.getRxBytes())
- .withMetric("net.in.errors", interfaceStats.getRxErrors())
- .withMetric("net.in.dropped", interfaceStats.getRxDropped())
- .withMetric("net.out.bytes", interfaceStats.getTxBytes())
- .withMetric("net.out.errors", interfaceStats.getTxErrors())
- .withMetric("net.out.dropped", interfaceStats.getTxDropped())
- .build();
- metrics.add(networkMetrics);
- });
-
- pushMetricsToContainer(context, metrics);
- }
-
- private void pushMetricsToContainer(NodeAgentContext context, List<DimensionMetrics> metrics) {
- StringBuilder params = new StringBuilder();
- try {
- for (DimensionMetrics dimensionMetrics : metrics) {
- params.append(dimensionMetrics.toSecretAgentReport());
- }
- String wrappedMetrics = "s:" + params.toString();
-
- // Push metrics to the metrics proxy in each container.
- // TODO Remove port selection logic when all hosted apps have upgraded to Vespa 7.
- int port = context.node().currentVespaVersion().map(version -> version.getMajor() == 6).orElse(false) ? 19091 : 19095;
- String[] command = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:" + port, "setExtraMetrics", wrappedMetrics};
- dockerOperations.executeCommandInContainerAsRoot(context, 5L, command);
- } catch (JsonProcessingException | DockerExecTimeoutException e) {
- context.log(logger, LogLevel.WARNING, "Failed to push metrics to container", e);
- }
-
- }
-
private Optional<Container> getContainer(NodeAgentContext context) {
if (containerState == ABSENT) return Optional.empty();
Optional<Container> container = dockerOperations.getContainer(context);
@@ -621,66 +505,6 @@ public class NodeAgentImpl implements NodeAgent {
return temp;
}
- class CpuUsageReporter {
- private static final double PERIOD_IN_NANOSECONDS = 1_000d * ContainerResources.CPU_PERIOD_US;
- private long containerKernelUsage = 0;
- private long totalContainerUsage = 0;
- private long totalSystemUsage = 0;
- private long throttledTime = 0;
- private long throttlingActivePeriods = 0;
- private long throttledPeriods = 0;
-
- private long deltaContainerKernelUsage;
- private long deltaContainerUsage;
- private long deltaSystemUsage;
- private long deltaThrottledTime;
- private long deltaThrottlingActivePeriods;
- private long deltaThrottledPeriods;
-
- private void updateCpuDeltas(ContainerStats.CpuStats cpuStats) {
- // Do not calculate delta during the first tick - that will result in a metric value that is
- // average since container start
- if (totalSystemUsage != 0) {
- deltaSystemUsage = cpuStats.getSystemCpuUsage() - totalSystemUsage;
- deltaContainerUsage = cpuStats.getTotalUsage() - totalContainerUsage;
- deltaContainerKernelUsage = cpuStats.getUsageInKernelMode() - containerKernelUsage;
- deltaThrottledTime = cpuStats.getThrottledTime() - throttledTime;
- deltaThrottlingActivePeriods = cpuStats.getThrottlingActivePeriods() - throttlingActivePeriods;
- deltaThrottledPeriods = cpuStats.getThrottledPeriods() - throttledPeriods;
- }
-
- totalSystemUsage = cpuStats.getSystemCpuUsage();
- totalContainerUsage = cpuStats.getTotalUsage();
- containerKernelUsage = cpuStats.getUsageInKernelMode();
- throttledTime = cpuStats.getThrottledTime();
- throttlingActivePeriods = cpuStats.getThrottlingActivePeriods();
- throttledPeriods = cpuStats.getThrottledPeriods();
- }
-
- /**
- * Returns the CPU usage ratio for the docker container that this NodeAgent is managing
- * in the time between the last two times updateCpuDeltas() was called. This is calculated
- * by dividing the CPU time used by the container with the CPU time used by the entire system.
- */
- double getCpuUsageRatio() {
- return deltaSystemUsage == 0 ? Double.NaN : (double) deltaContainerUsage / deltaSystemUsage;
- }
-
- double getCpuKernelUsageRatio() {
- return deltaSystemUsage == 0 ? Double.NaN : (double) deltaContainerKernelUsage / deltaSystemUsage;
- }
-
- double getThrottledTimeRate() {
- return deltaThrottlingActivePeriods == 0 ? Double.NaN :
- (double) deltaThrottledPeriods / deltaThrottlingActivePeriods;
- }
-
- double getThrottledCpuTimeRate() {
- return deltaThrottlingActivePeriods == 0 ? Double.NaN :
- deltaThrottledTime / (PERIOD_IN_NANOSECONDS * deltaThrottlingActivePeriods);
- }
- }
-
// TODO: Also skip orchestration if we're downgrading in test/staging
// How to implement:
// - test/staging: We need to figure out whether we're in test/staging, zone is available in Environment
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java
index a5daab8dcfd..956302dcdcc 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java
@@ -19,4 +19,7 @@ public interface NodeAgentScheduler {
* @return True if node agent has converged to the desired state
*/
boolean setFrozen(boolean frozen, Duration timeout);
+
+ /** @return the last scheduled context or a default value */
+ NodeAgentContext currentContext();
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java
index cc31374669c..5bd3d7800e6 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java
@@ -19,7 +19,6 @@ import java.util.stream.Collectors;
*/
@ThreadSafe
public class DebugHandlerHelper implements NodeAdminDebugHandler {
- private Object monitor = new Object();
private final ConcurrentMap<String, Supplier<Object>> suppliers = new ConcurrentHashMap<>();
public void addThreadSafeSupplier(String name, Supplier<Object> threadSafeSupplier) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java
index 5400c19d63e..3bcd806bc85 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java
@@ -14,7 +14,7 @@ import java.util.logging.Logger;
* @author hakonhall
*/
public class StoredBoolean {
- private static Logger logger = Logger.getLogger(StoredBoolean.class.getName());
+ private static final Logger logger = Logger.getLogger(StoredBoolean.class.getName());
private final UnixPath path;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java
index 113af76972b..e8c22184406 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java
@@ -14,8 +14,8 @@ import java.util.Map;
public class Templar {
private final String template;
- private String prefix = "<%=";
- private String suffix = "%>";
+ private static final String prefix = "<%=";
+ private static final String suffix = "%>";
private final Map<String, String> settings = new HashMap<>();
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java
index 376fda1d2dc..cf6c6c432f4 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java
@@ -60,7 +60,7 @@ public class UnixPath {
public Optional<String> readUtf8FileIfExists() {
try {
- return Optional.of(new String(Files.readAllBytes(path), StandardCharsets.UTF_8));
+ return Optional.of(Files.readString(path));
} catch (NoSuchFileException ignored) {
return Optional.empty();
} catch (IOException e) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
index ba1952545a0..85a7c065a86 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
@@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
@@ -37,7 +36,7 @@ public class Yum {
.map(formatter -> "%{" + formatter + "}")
.collect(Collectors.joining("\\n"));
private static final Function<YumPackageName.Builder, List<Function<String, YumPackageName.Builder>>>
- PACKAGE_NAME_BUILDERS_GENERATOR = builder -> Arrays.asList(
+ PACKAGE_NAME_BUILDERS_GENERATOR = builder -> List.of(
builder::setName, builder::setEpoch, builder::setVersion, builder::setRelease, builder::setArchitecture);
@@ -183,7 +182,7 @@ public class Yum {
return new GenericYumCommand(
terminal,
yumCommand,
- Arrays.asList(packages),
+ List.of(packages),
noopPattern);
}
@@ -209,9 +208,8 @@ public class Yum {
}
}
- @SuppressWarnings("unchecked")
public GenericYumCommand enableRepos(String... repos) {
- enabledRepo.addAll(Arrays.asList(repos));
+ enabledRepo.addAll(List.of(repos));
return this;
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java
deleted file mode 100644
index cdf67871a1a..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java
+++ /dev/null
@@ -1,101 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.util;
-
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.hosted.node.admin.task.util.file.FileWriter;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-/**
- * Helper class to generate and write the secret-agent check config files.
- *
- * @author freva
- */
-public class SecretAgentCheckConfig {
- private final String id;
- private final int interval;
- private final Path checkExecutable;
- private final String[] arguments;
- private String user = "nobody";
- private final Map<String, Object> tags = new LinkedHashMap<>();
-
- public SecretAgentCheckConfig(String id, int interval, Path checkExecutable, String... arguments) {
- this.id = id;
- this.interval = interval;
- this.checkExecutable = checkExecutable;
- this.arguments = arguments;
- }
-
- public SecretAgentCheckConfig withRunAsUser(String user) {
- this.user = user;
- return this;
- }
-
- public SecretAgentCheckConfig withTag(String tagKey, Object tagValue) {
- tags.put(tagKey, tagValue);
- return this;
- }
-
- public SecretAgentCheckConfig withTags(Map<String, Object> tags) {
- this.tags.clear();
- this.tags.putAll(tags);
- return this;
- }
-
- public void setTags(Map<String, Object> tags) {
- this.tags.clear();
- this.tags.putAll(tags);
- }
-
- public void writeTo(Path yamasAgentDirectory) throws IOException {
- Files.createDirectories(yamasAgentDirectory);
- Path scheduleFilePath = yamasAgentDirectory.resolve(id + ".yaml");
- Files.write(scheduleFilePath, render().getBytes());
- }
-
- public FileWriter getFileWriterTo(Path destinationPath) {
- return new FileWriter(destinationPath, this::render);
- }
-
- public String render() {
- StringBuilder stringBuilder = new StringBuilder()
- .append("- id: ").append(id).append("\n")
- .append(" interval: ").append(interval).append("\n")
- .append(" user: ").append(user).append("\n")
- .append(" check: ").append(checkExecutable.toFile()).append("\n");
-
- if (arguments.length > 0) {
- stringBuilder.append(" args:\n");
- for (String arg : arguments) {
- stringBuilder.append(" - ").append(arg).append("\n");
- }
- }
-
- if (!tags.isEmpty()) {
- stringBuilder.append(" tags:\n");
- tags.forEach((key, value) ->
- stringBuilder.append(" ").append(key).append(": ").append(value).append("\n"));
- }
-
- return stringBuilder.toString();
- }
-
- // TODO: Change role dimension to nodeType?
- public static String nodeTypeToRole(NodeType nodeType) {
- switch (nodeType) {
- case tenant: return "tenants";
- case host: return "docker";
- case proxy: return "routing";
- case proxyhost: return "routinghost";
- case config: return "configserver";
- case confighost: return "configserverhost";
- case controller: return "controller";
- case controllerhost: return "controllerhost";
- default: throw new IllegalArgumentException("Unknown node type " + nodeType);
- }
- }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/package-info.java
deleted file mode 100644
index 56cb135e723..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/package-info.java
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage
-package com.yahoo.vespa.hosted.node.admin.util;
-
-import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
index 57b18606def..d034d3c1cd0 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
@@ -2,21 +2,9 @@
package com.yahoo.vespa.hosted.node.admin.maintenance;
import com.google.common.collect.ImmutableSet;
-import com.yahoo.component.Version;
-import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.config.provision.RegionName;
-import com.yahoo.config.provision.zone.ZoneApi;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
-import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
import com.yahoo.vespa.test.file.TestFileSystem;
import org.junit.After;
@@ -35,162 +23,15 @@ import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import static com.yahoo.yolean.Exceptions.uncheck;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
/**
* @author dybis
*/
@RunWith(Enclosed.class)
public class StorageMaintainerTest {
- private static final DockerOperations docker = mock(DockerOperations.class);
-
- public static class SecretAgentCheckTests {
- private final StorageMaintainer storageMaintainer = new StorageMaintainer(null, docker, null, null);
-
- @Test
- public void tenant() {
- Path path = executeAs(NodeType.tenant);
-
- assertChecks(path, "athenz-certificate-expiry", "host-life",
- "system-coredumps-processing", "vespa", "vespa-health");
-
- // All dimensions for vespa metrics should be set by metricsproxy
- assertCheckEnds(path.resolve("vespa.yaml"),
- " args:\n" +
- " - all\n");
-
- // For non vespa metrics, we need to set all the dimensions ourselves
- assertCheckEnds(path.resolve("host-life.yaml"),
- "tags:\n" +
- " namespace: Vespa\n" +
- " role: tenants\n" +
- " zone: prod.us-north-1\n" +
- " vespaVersion: 6.305.12\n" +
- " state: active\n" +
- " parentHostname: host123.test.domain.tld\n" +
- " tenantName: tenant\n" +
- " app: application.instance\n" +
- " applicationName: application\n" +
- " instanceName: instance\n" +
- " applicationId: tenant.application.instance\n" +
- " clustertype: clusterType\n" +
- " clusterid: clusterId\n");
- }
-
- @Test
- public void proxy() {
- Path path = executeAs(NodeType.proxy);
-
- assertChecks(path, "athenz-certificate-expiry", "host-life", "routing-configage",
- "ssl-status", "system-coredumps-processing", "vespa", "vespa-health");
-
- // All dimensions for vespa metrics should be set by the source
- assertCheckEnds(path.resolve("vespa.yaml"),
- " args:\n" +
- " - all\n");
-
- // For non vespa metrics, we need to set all the dimensions ourselves
- assertCheckEnds(path.resolve("host-life.yaml"),
- "tags:\n" +
- " namespace: Vespa\n" +
- " role: routing\n" +
- " zone: prod.us-north-1\n" +
- " vespaVersion: 6.305.12\n" +
- " state: active\n" +
- " parentHostname: host123.test.domain.tld\n" +
- " tenantName: tenant\n" +
- " app: application.instance\n" +
- " applicationName: application\n" +
- " instanceName: instance\n" +
- " applicationId: tenant.application.instance\n" +
- " clustertype: clusterType\n" +
- " clusterid: clusterId\n");
- }
-
- @Test
- public void configserver() {
- Path path = executeAs(NodeType.config);
-
- assertChecks(path, "athenz-certificate-expiry", "configserver", "configserver-logd", "host-life",
- "system-coredumps-processing", "zkbackupage");
-
- assertCheckEnds(path.resolve("configserver.yaml"),
- " tags:\n" +
- " namespace: Vespa\n" +
- " role: configserver\n" +
- " zone: prod.us-north-1\n" +
- " vespaVersion: 6.305.12\n");
- }
-
- @Test
- public void controller() {
- Path path = executeAs(NodeType.controller);
-
- assertChecks(path, "athenz-certificate-expiry", "controller", "controller-logd", "host-life",
- "system-coredumps-processing", "vespa", "vespa-health", "zkbackupage");
-
-
- // Do not set namespace for vespa metrics. WHY?
- assertCheckEnds(path.resolve("vespa.yaml"),
- " tags:\n" +
- " role: controller\n" +
- " zone: prod.us-north-1\n" +
- " vespaVersion: 6.305.12\n");
-
- assertCheckEnds(path.resolve("controller.yaml"),
- " tags:\n" +
- " namespace: Vespa\n" +
- " role: controller\n" +
- " zone: prod.us-north-1\n" +
- " vespaVersion: 6.305.12\n");
- }
-
- private Path executeAs(NodeType nodeType) {
- ZoneApi zone = mock(ZoneApi.class);
- when(zone.getId()).thenReturn(ZoneId.from(Environment.prod, RegionName.from("us-north-1")));
-
- NodeSpec nodeSpec = new NodeSpec.Builder()
- .hostname("host123-5.test.domain.tld")
- .type(nodeType)
- .state(NodeState.active)
- .parentHostname("host123.test.domain.tld")
- .owner(new NodeOwner("tenant", "application", "instance"))
- .membership(new NodeMembership("clusterType", "clusterId", null, 0, false))
- .currentVespaVersion(Version.fromString("6.305.12"))
- .flavor("d-2-8-50")
- .canonicalFlavor("d-2-8-50")
- .build();
- NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec)
- .fileSystem(TestFileSystem.create())
- .zone(zone)
- .build();
- Path path = context.pathOnHostFromPathInNode("/etc/yamas-agent");
- uncheck(() -> Files.createDirectories(path));
- storageMaintainer.writeMetricsConfig(context);
- return path;
- }
-
- private void assertCheckEnds(Path checkPath, String contentsEnd) {
- String contents = new UnixPath(checkPath).readUtf8File();
- assertTrue(contents, contents.endsWith(contentsEnd));
- }
-
- private void assertChecks(Path checksPath, String... checkNames) {
- List<String> expectedChecks = Stream.of(checkNames).sorted().collect(Collectors.toList());
- List<String> actualChecks = FileFinder.files(checksPath).stream()
- .map(FileFinder.FileAttributes::filename)
- .map(filename -> filename.replaceAll("\\.yaml$", ""))
- .sorted()
- .collect(Collectors.toList());
- assertEquals(expectedChecks, actualChecks);
- }
- }
public static class DiskUsageTests {
@@ -198,7 +39,7 @@ public class StorageMaintainerTest {
@Test
public void testDiskUsed() throws IOException {
- StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, docker, null, null);
+ StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null);
FileSystem fileSystem = TestFileSystem.create();
NodeAgentContext context = new NodeAgentContextImpl.Builder("host-1.domain.tld").fileSystem(fileSystem).build();
Files.createDirectories(context.pathOnHostFromPathInNode("/"));
@@ -212,7 +53,7 @@ public class StorageMaintainerTest {
@Test
public void testNonExistingDiskUsed() {
- StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, docker, null, null);
+ StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null);
long usedBytes = storageMaintainer.getDiskUsedInBytes(null, Paths.get("/fake/path"));
assertEquals(0L, usedBytes);
}
@@ -244,7 +85,7 @@ public class StorageMaintainerTest {
// Archive container-1
- StorageMaintainer storageMaintainer = new StorageMaintainer(null, docker, null, pathToArchiveDir);
+ StorageMaintainer storageMaintainer = new StorageMaintainer(null, null, pathToArchiveDir);
storageMaintainer.archiveNodeStorage(context1);
// container-1 should be gone from container-storage
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 46af7e7bafd..c0b032bc4d4 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -9,12 +9,8 @@ import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.dockerapi.Container;
import com.yahoo.vespa.hosted.dockerapi.ContainerName;
import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
-import com.yahoo.vespa.hosted.dockerapi.ContainerStats;
import com.yahoo.vespa.hosted.dockerapi.exception.DockerException;
-import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
@@ -28,21 +24,12 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
import org.junit.Test;
import org.mockito.InOrder;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import static com.yahoo.yolean.Exceptions.uncheck;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
@@ -76,7 +63,6 @@ public class NodeAgentImplTest {
private final NodeRepository nodeRepository = mock(NodeRepository.class);
private final Orchestrator orchestrator = mock(Orchestrator.class);
private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class);
- private final Metrics metrics = new Metrics();
private final AclMaintainer aclMaintainer = mock(AclMaintainer.class);
private final HealthChecker healthChecker = mock(HealthChecker.class);
private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class);
@@ -152,7 +138,7 @@ public class NodeAgentImplTest {
inOrder.verify(dockerOperations, never()).startServices(eq(context));
inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
- nodeAgent.stopForHostSuspension();
+ nodeAgent.stopForHostSuspension(context);
nodeAgent.doConverge(context);
inOrder.verify(dockerOperations, never()).startServices(eq(context));
inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services
@@ -162,7 +148,7 @@ public class NodeAgentImplTest {
inOrder.verify(dockerOperations, never()).startServices(eq(context));
inOrder.verify(dockerOperations, never()).resumeNode(eq(context));
- nodeAgent.stopForHostSuspension();
+ nodeAgent.stopForHostSuspension(context);
nodeAgent.doConverge(context);
inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), any(), any());
inOrder.verify(dockerOperations, times(1)).startContainer(eq(context));
@@ -638,81 +624,6 @@ public class NodeAgentImplTest {
}
@Test
- @SuppressWarnings("unchecked")
- public void testGetRelevantMetrics() throws Exception {
- String json = Files.readString(Paths.get("src/test/resources/docker.stats.json"));
- ContainerStats stats2 = ContainerStats.fromJson(json);
- ContainerStats stats1 = ContainerStats.fromJson(json.replace("\"cpu_stats\"", "\"cpu_stats2\"").replace("\"precpu_stats\"", "\"cpu_stats\""));
-
- NodeOwner owner = new NodeOwner("tester", "testapp", "testinstance");
- NodeMembership membership = new NodeMembership("clustType", "clustId", "grp", 3, false);
- final NodeSpec node = nodeBuilder
- .wantedDockerImage(dockerImage)
- .currentDockerImage(dockerImage)
- .state(NodeState.active)
- .currentVespaVersion(vespaVersion)
- .owner(owner)
- .membership(membership)
- .memoryGb(2)
- .allowedToBeDown(true)
- .parentHostname("parent.host.name.yahoo.com")
- .build();
-
- NodeAgentContext context = createContext(node);
- NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
-
- when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node));
- when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(39625000000L));
- when(dockerOperations.getContainerStats(eq(context)))
- .thenReturn(Optional.of(stats1))
- .thenReturn(Optional.of(stats2));
-
- List<String> expectedMetrics = Stream.of(0, 1)
- .map(i -> Paths.get("src/test/resources/expected.container.system.metrics." + i + ".txt"))
- .map(path -> uncheck(() -> Files.readString(path)))
- .map(content -> content.replaceAll("\\s", "").replaceAll("\\n", ""))
- .collect(Collectors.toList());
- int[] counter = {0};
-
- doAnswer(invocation -> {
- NodeAgentContext calledContainerName = (NodeAgentContext) invocation.getArguments()[0];
- long calledTimeout = (long) invocation.getArguments()[1];
- String[] calledCommand = new String[invocation.getArguments().length - 2];
- System.arraycopy(invocation.getArguments(), 2, calledCommand, 0, calledCommand.length);
- calledCommand[calledCommand.length - 1] = calledCommand[calledCommand.length - 1]
- .replaceAll("\"timestamp\":\\d+", "\"timestamp\":0")
- .replaceAll("([0-9]+\\.[0-9]{1,3})([0-9]*)", "$1"); // Only keep the first 3 decimals
-
- assertEquals(context, calledContainerName);
- assertEquals(5L, calledTimeout);
- String[] expectedCommand = {"vespa-rpc-invoke", "-t", "2", "tcp/localhost:19095",
- "setExtraMetrics", expectedMetrics.get(counter[0])};
- assertArrayEquals("Ivocation #" + counter[0], expectedCommand, calledCommand);
- counter[0]++;
- return null;
- }).when(dockerOperations).executeCommandInContainerAsRoot(any(), any(), any());
-
- nodeAgent.updateContainerNodeMetrics();
- nodeAgent.updateContainerNodeMetrics();
- }
-
- @Test
- public void testGetRelevantMetricsForReadyNode() {
- final NodeSpec node = nodeBuilder
- .state(NodeState.ready)
- .build();
-
- NodeAgentContext context = createContext(node);
- NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
-
- when(dockerOperations.getContainerStats(eq(context))).thenReturn(Optional.empty());
-
- nodeAgent.updateContainerNodeMetrics();
-
- assertEquals(List.of(), metrics.getDefaultMetrics());
- }
-
- @Test
public void testRunningConfigServer() {
final NodeSpec node = nodeBuilder
.type(NodeType.config)
@@ -747,8 +658,6 @@ public class NodeAgentImplTest {
private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) {
mockGetContainer(dockerImage, isRunning);
- doNothing().when(storageMaintainer).writeMetricsConfig(any());
-
return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations,
storageMaintainer, flagSource, Optional.of(credentialsMaintainer), Optional.of(aclMaintainer),
Optional.of(healthChecker));
@@ -776,8 +685,6 @@ public class NodeAgentImplTest {
}
private NodeAgentContext createContext(NodeSpec nodeSpec) {
- NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec).build();
- when(contextSupplier.currentContext()).thenReturn(context);
- return context;
+ return new NodeAgentContextImpl.Builder(nodeSpec).build();
}
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java
index 333cb81f9d4..e66b3a7aed2 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java
@@ -11,7 +11,7 @@ import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
-import java.util.Arrays;
+import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -31,7 +31,7 @@ public class ProcessFactoryImplTest {
@Test
public void testSpawn() {
CommandLine commandLine = mock(CommandLine.class);
- when(commandLine.getArguments()).thenReturn(Arrays.asList("program"));
+ when(commandLine.getArguments()).thenReturn(List.of("program"));
when(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()).thenReturn(true);
when(commandLine.programName()).thenReturn("program");
Path outputPath;
@@ -56,8 +56,8 @@ public class ProcessFactoryImplTest {
public void testSpawnWithPersistentOutputFile() {
class TemporaryFile implements AutoCloseable {
- Path path;
- TemporaryFile() {
+ private final Path path;
+ private TemporaryFile() {
String outputFileName = ProcessFactoryImplTest.class.getSimpleName() + "-temporary-test-file.out";
FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute(
PosixFilePermissions.fromString("rw-------"));
@@ -68,7 +68,7 @@ public class ProcessFactoryImplTest {
try (TemporaryFile outputPath = new TemporaryFile()) {
CommandLine commandLine = mock(CommandLine.class);
- when(commandLine.getArguments()).thenReturn(Arrays.asList("program"));
+ when(commandLine.getArguments()).thenReturn(List.of("program"));
when(commandLine.programName()).thenReturn("program");
when(commandLine.getOutputFile()).thenReturn(Optional.of(outputPath.path));
try (ChildProcess2Impl child = processFactory.spawn(commandLine)) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java
deleted file mode 100644
index 30263403757..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.util;
-
-import com.yahoo.config.provision.NodeType;
-import org.junit.Test;
-
-import java.nio.file.Paths;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * @author freva
- */
-public class SecretAgentCheckConfigTest {
-
- @Test
- public void generateFullSecretAgentScheduleTest() {
- SecretAgentCheckConfig scheduleMaker = new SecretAgentCheckConfig("system-checks", 60,
- Paths.get("/some/test"), "arg1", "arg2 with space")
- .withTag("tenantName", "vespa")
- .withTag("applicationName", "canary-docker")
- .withTag("instanceName", "default")
- .withTag("applicationId", "vespa.canary-docker.default")
- .withTag("app", "canary-docker.default")
- .withTag("clustertype", "container")
- .withTag("clusterid", "canary")
- .withTag("vespaVersion", "6.13.37")
- .withTag("role", "tenants")
- .withTag("flavor", "docker")
- .withTag("state", "active")
- .withTag("zone", "test.us-west-5");
-
- assertEquals(
- "- id: system-checks\n" +
- " interval: 60\n" +
- " user: nobody\n" +
- " check: /some/test\n" +
- " args:\n" +
- " - arg1\n" +
- " - arg2 with space\n" +
- " tags:\n" +
- " tenantName: vespa\n" +
- " applicationName: canary-docker\n" +
- " instanceName: default\n" +
- " applicationId: vespa.canary-docker.default\n" +
- " app: canary-docker.default\n" +
- " clustertype: container\n" +
- " clusterid: canary\n" +
- " vespaVersion: 6.13.37\n" +
- " role: tenants\n" +
- " flavor: docker\n" +
- " state: active\n" +
- " zone: test.us-west-5\n", scheduleMaker.render());
- }
-
- @Test
- public void generateMinimalSecretAgentScheduleTest() {
- SecretAgentCheckConfig scheduleMaker = new SecretAgentCheckConfig("system-checks", 60,
- Paths.get("/some/test"));
-
- assertEquals(
- "- id: system-checks\n" +
- " interval: 60\n" +
- " user: nobody\n" +
- " check: /some/test\n", scheduleMaker.render());
- }
-
- @Test
- public void generateSecretAgentScheduleWithDifferentUserTest() {
- SecretAgentCheckConfig scheduleMaker = new SecretAgentCheckConfig("system-checks", 60,
- Paths.get("/some/test")).withRunAsUser("barfoo");
-
- assertEquals(
- "- id: system-checks\n" +
- " interval: 60\n" +
- " user: barfoo\n" +
- " check: /some/test\n", scheduleMaker.render());
- }
-
- @Test
- public void supportsAllNodeTypes() {
- for (NodeType nodeType : NodeType.values()) {
- assertNotNull(SecretAgentCheckConfig.nodeTypeToRole(nodeType));
- }
- }
-
-}
diff --git a/node-admin/src/test/resources/docker.stats.json b/node-admin/src/test/resources/docker.stats.json
deleted file mode 100644
index 5b42d9a2428..00000000000
--- a/node-admin/src/test/resources/docker.stats.json
+++ /dev/null
@@ -1,376 +0,0 @@
-{
- "read":"2016-10-05T07:28:17.228361751Z",
- "precpu_stats":{
- "cpu_usage":{
- "total_usage":332026268600,
- "percpu_usage":[
- 46767331190,
- 46637593621,
- 36196010351,
- 38846420953,
- 44237804850,
- 35751912062,
- 44546685143,
- 39042510430
- ],
- "usage_in_kernelmode":44040000000,
- "usage_in_usermode":158940000000
- },
- "system_cpu_usage":5876874910000000,
- "throttling_data":{
- "periods":820694,
- "throttled_periods":177731,
- "throttled_time":81891944744550
- }
- },
- "cpu_stats":{
- "cpu_usage":{
- "total_usage":332131163600,
- "percpu_usage":[
- 46774042376,
- 46639549207,
- 36204341756,
- 38879138416,
- 44256253747,
- 35760081676,
- 44567860460,
- 39049895962
- ],
- "usage_in_kernelmode":44106083850,
- "usage_in_usermode":158950000000
- },
- "system_cpu_usage":5876882680000000,
- "throttling_data":{
- "periods":821264,
- "throttled_periods":178201,
- "throttled_time":82181944744550
- }
- },
- "memory_stats":{
- "usage":1752707072,
- "max_usage":1818116096,
- "stats":{
- "active_anon":1326051328,
- "active_file":188919808,
- "cache":678965248,
- "hierarchical_memory_limit":4294967296,
- "hierarchical_memsw_limit":8589934592,
- "inactive_anon":0,
- "inactive_file":237735936,
- "mapped_file":62976000,
- "pgfault":3102812,
- "pgmajfault":1403,
- "pgpgin":1691151,
- "pgpgout":1263244,
- "rss":1326026752,
- "rss_huge":0,
- "swap":0,
- "total_active_anon":1326051328,
- "total_active_file":188919808,
- "total_cache":426680320,
- "total_inactive_anon":0,
- "total_inactive_file":237735936,
- "total_mapped_file":62976000,
- "total_pgfault":3102812,
- "total_pgmajfault":1403,
- "total_pgpgin":1691151,
- "total_pgpgout":1263244,
- "total_rss":1326026752,
- "total_rss_huge":0,
- "total_swap":0,
- "total_unevictable":0,
- "unevictable":0
- },
- "failcnt":0,
- "limit":4294967296
- },
- "blkio_stats":{
- "io_service_bytes_recursive":[
- {
- "major":252,
- "minor":0,
- "op":"Read",
- "value":53248
- },
- {
- "major":252,
- "minor":0,
- "op":"Write",
- "value":602112
- },
- {
- "major":252,
- "minor":0,
- "op":"Sync",
- "value":0
- },
- {
- "major":252,
- "minor":0,
- "op":"Async",
- "value":655360
- },
- {
- "major":252,
- "minor":0,
- "op":"Total",
- "value":655360
- },
- {
- "major":7,
- "minor":0,
- "op":"Read",
- "value":308224
- },
- {
- "major":7,
- "minor":0,
- "op":"Write",
- "value":573440
- },
- {
- "major":7,
- "minor":0,
- "op":"Sync",
- "value":0
- },
- {
- "major":7,
- "minor":0,
- "op":"Async",
- "value":881664
- },
- {
- "major":7,
- "minor":0,
- "op":"Total",
- "value":881664
- },
- {
- "major":253,
- "minor":0,
- "op":"Read",
- "value":308224
- },
- {
- "major":253,
- "minor":0,
- "op":"Write",
- "value":573440
- },
- {
- "major":253,
- "minor":0,
- "op":"Sync",
- "value":0
- },
- {
- "major":253,
- "minor":0,
- "op":"Async",
- "value":881664
- },
- {
- "major":253,
- "minor":0,
- "op":"Total",
- "value":881664
- },
- {
- "major":253,
- "minor":3,
- "op":"Read",
- "value":343847936
- },
- {
- "major":253,
- "minor":3,
- "op":"Write",
- "value":786432
- },
- {
- "major":253,
- "minor":3,
- "op":"Sync",
- "value":131072
- },
- {
- "major":253,
- "minor":3,
- "op":"Async",
- "value":344503296
- },
- {
- "major":253,
- "minor":3,
- "op":"Total",
- "value":344634368
- }
- ],
- "io_serviced_recursive":[
- {
- "major":252,
- "minor":0,
- "op":"Read",
- "value":13
- },
- {
- "major":252,
- "minor":0,
- "op":"Write",
- "value":147
- },
- {
- "major":252,
- "minor":0,
- "op":"Sync",
- "value":0
- },
- {
- "major":252,
- "minor":0,
- "op":"Async",
- "value":160
- },
- {
- "major":252,
- "minor":0,
- "op":"Total",
- "value":160
- },
- {
- "major":7,
- "minor":0,
- "op":"Read",
- "value":37
- },
- {
- "major":7,
- "minor":0,
- "op":"Write",
- "value":124
- },
- {
- "major":7,
- "minor":0,
- "op":"Sync",
- "value":0
- },
- {
- "major":7,
- "minor":0,
- "op":"Async",
- "value":161
- },
- {
- "major":7,
- "minor":0,
- "op":"Total",
- "value":161
- },
- {
- "major":253,
- "minor":0,
- "op":"Read",
- "value":37
- },
- {
- "major":253,
- "minor":0,
- "op":"Write",
- "value":124
- },
- {
- "major":253,
- "minor":0,
- "op":"Sync",
- "value":0
- },
- {
- "major":253,
- "minor":0,
- "op":"Async",
- "value":161
- },
- {
- "major":253,
- "minor":0,
- "op":"Total",
- "value":161
- },
- {
- "major":253,
- "minor":3,
- "op":"Read",
- "value":11812
- },
- {
- "major":253,
- "minor":3,
- "op":"Write",
- "value":142
- },
- {
- "major":253,
- "minor":3,
- "op":"Sync",
- "value":2
- },
- {
- "major":253,
- "minor":3,
- "op":"Async",
- "value":11952
- },
- {
- "major":253,
- "minor":3,
- "op":"Total",
- "value":11954
- }
- ],
- "io_queue_recursive":[
-
- ],
- "io_service_time_recursive":[
-
- ],
- "io_wait_time_recursive":[
-
- ],
- "io_merged_recursive":[
-
- ],
- "io_time_recursive":[
-
- ],
- "sectors_recursive":[
-
- ]
- },
- "pids_stats":{
-
- },
- "networks":{
- "eth0":{
- "rx_bytes":19499270,
- "rx_packets":58913,
- "rx_errors":55,
- "rx_dropped":4,
- "tx_bytes":20303455,
- "tx_packets":62319,
- "tx_errors":3,
- "tx_dropped":13
- },
- "eth1":{
- "rx_bytes":3245766,
- "rx_packets":23462,
- "rx_errors":0,
- "rx_dropped":0,
- "tx_bytes":54246745,
- "tx_packets":34562,
- "tx_errors":0,
- "tx_dropped":0
- }
- }
-} \ No newline at end of file
diff --git a/node-admin/src/test/resources/expected.container.system.metrics.0.txt b/node-admin/src/test/resources/expected.container.system.metrics.0.txt
deleted file mode 100644
index ea6036ce2ea..00000000000
--- a/node-admin/src/test/resources/expected.container.system.metrics.0.txt
+++ /dev/null
@@ -1,78 +0,0 @@
-s:
-{
- "application": "vespa.node",
- "dimensions": {
- "host": "host1.test.yahoo.com",
- "orchestratorState":"ALLOWED_TO_BE_DOWN",
- "parentHostname": "parent.host.name.yahoo.com",
- "role": "tenants",
- "state": "active"
- },
- "metrics": {
- "cpu.vcpus": 2.0,
- "disk.limit": 250000000000,
- "disk.used": 39625000000,
- "disk.util": 15.85,
- "mem.limit": 4294967296,
- "mem.used": 1073741824,
- "mem.util": 25.0,
- "mem_total.used": 1752707072,
- "mem_total.util": 40.808
- },
- "routing": {
- "yamas": {
- "namespaces": ["Vespa"]
- }
- },
- "timestamp": 0
-}
-{
- "application": "vespa.node",
- "dimensions": {
- "host": "host1.test.yahoo.com",
- "interface": "eth0",
- "orchestratorState":"ALLOWED_TO_BE_DOWN",
- "parentHostname": "parent.host.name.yahoo.com",
- "role": "tenants",
- "state": "active"
- },
- "metrics": {
- "net.in.bytes": 19499270,
- "net.in.dropped": 4,
- "net.in.errors": 55,
- "net.out.bytes": 20303455,
- "net.out.dropped": 13,
- "net.out.errors": 3
- },
- "routing": {
- "yamas": {
- "namespaces": ["Vespa"]
- }
- },
- "timestamp": 0
-}
-{
- "application": "vespa.node",
- "dimensions": {
- "host": "host1.test.yahoo.com",
- "interface": "eth1",
- "orchestratorState":"ALLOWED_TO_BE_DOWN",
- "parentHostname": "parent.host.name.yahoo.com",
- "role": "tenants",
- "state": "active"
- },
- "metrics": {
- "net.in.bytes": 3245766,
- "net.in.dropped": 0,
- "net.in.errors": 0,
- "net.out.bytes": 54246745,
- "net.out.dropped": 0,
- "net.out.errors": 0
- },
- "routing": {
- "yamas": {
- "namespaces": ["Vespa"]
- }
- },
- "timestamp": 0
-} \ No newline at end of file
diff --git a/node-admin/src/test/resources/expected.container.system.metrics.1.txt b/node-admin/src/test/resources/expected.container.system.metrics.1.txt
deleted file mode 100644
index 54d4d36c7d0..00000000000
--- a/node-admin/src/test/resources/expected.container.system.metrics.1.txt
+++ /dev/null
@@ -1,82 +0,0 @@
-s:
-{
- "application": "vespa.node",
- "dimensions": {
- "host": "host1.test.yahoo.com",
- "orchestratorState":"ALLOWED_TO_BE_DOWN",
- "parentHostname": "parent.host.name.yahoo.com",
- "role": "tenants",
- "state": "active"
- },
- "metrics": {
- "cpu.sys.util": 3.402,
- "cpu.throttled_cpu_time.rate": 5.087,
- "cpu.throttled_time.rate": 0.824,
- "cpu.util": 5.4,
- "cpu.vcpus": 2.0,
- "disk.limit": 250000000000,
- "disk.used": 39625000000,
- "disk.util": 15.85,
- "mem.limit": 4294967296,
- "mem.used": 1073741824,
- "mem.util": 25.0,
- "mem_total.used": 1752707072,
- "mem_total.util": 40.808
- },
- "routing": {
- "yamas": {
- "namespaces": ["Vespa"]
- }
- },
- "timestamp": 0
-}
-{
- "application": "vespa.node",
- "dimensions": {
- "host": "host1.test.yahoo.com",
- "interface": "eth0",
- "orchestratorState":"ALLOWED_TO_BE_DOWN",
- "parentHostname": "parent.host.name.yahoo.com",
- "role": "tenants",
- "state": "active"
- },
- "metrics": {
- "net.in.bytes": 19499270,
- "net.in.dropped": 4,
- "net.in.errors": 55,
- "net.out.bytes": 20303455,
- "net.out.dropped": 13,
- "net.out.errors": 3
- },
- "routing": {
- "yamas": {
- "namespaces": ["Vespa"]
- }
- },
- "timestamp": 0
-}
-{
- "application": "vespa.node",
- "dimensions": {
- "host": "host1.test.yahoo.com",
- "interface": "eth1",
- "orchestratorState":"ALLOWED_TO_BE_DOWN",
- "parentHostname": "parent.host.name.yahoo.com",
- "role": "tenants",
- "state": "active"
- },
- "metrics": {
- "net.in.bytes": 3245766,
- "net.in.dropped": 0,
- "net.in.errors": 0,
- "net.out.bytes": 54246745,
- "net.out.dropped": 0,
- "net.out.errors": 0
- },
- "routing": {
- "yamas": {
- "namespaces": ["Vespa"]
- }
- },
- "timestamp": 0
-} \ No newline at end of file