diff options
Diffstat (limited to 'node-admin')
24 files changed, 432 insertions, 310 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java index f1218efa67c..1b4ddf6aedd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java @@ -16,10 +16,9 @@ import java.util.Map; * * @author hmusum */ -@FunctionalInterface public interface ContainerEnvironmentResolver { - String createSettings(NodeSpec node); + String createSettings(Environment environment, NodeSpec node); class ContainerEnvironmentSettings { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java new file mode 100644 index 00000000000..38f57c9544a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java @@ -0,0 +1,18 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.component; + +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; + +/** + * @author hmusum + */ +public class DefaultContainerEnvironmentResolver implements ContainerEnvironmentResolver { + + public String createSettings(Environment environment, NodeSpec node) { + return new ContainerEnvironmentSettings() + .set("nodeType", node.getNodeType()) + .set("cloud", environment.getCloud()) + .build(); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java index aaadd3cb24e..857f796fb5c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java @@ -307,7 +307,7 @@ public class Environment { Optional.ofNullable(pathResolver).orElseGet(PathResolver::new), logstashNodes, nodeType, - Optional.ofNullable(containerEnvironmentResolver).orElseGet(() -> node -> ""), + Optional.ofNullable(containerEnvironmentResolver).orElseGet(DefaultContainerEnvironmentResolver::new), certificateDnsSuffix, ztsUri, nodeAthenzIdentity, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ZoneId.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ZoneId.java deleted file mode 100644 index 02f6da2e0e3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ZoneId.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; - -import java.util.Objects; - -/** - * @author freva - */ -public class ZoneId { - private final SystemName systemName; - private final Environment environment; - private final RegionName regionName; - - public ZoneId(SystemName systemName, Environment environment, RegionName regionName) { - this.systemName = systemName; - this.environment = environment; - this.regionName = regionName; - } - - public SystemName systemName() { - return systemName; - } - - public Environment environment() { - return environment; - } - - public RegionName regionName() { - return regionName; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ZoneId zoneId = (ZoneId) o; - return systemName == zoneId.systemName && - environment == zoneId.environment && - Objects.equals(regionName, zoneId.regionName); - } - - @Override - public int hashCode() { - return Objects.hash(systemName, environment, regionName); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 7a83f00e297..f1fc6776380 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.node.admin.component.ContainerEnvironmentResolver; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; @@ -46,19 +46,13 @@ public class DockerOperationsImpl implements DockerOperations { private static final String IPV4_NPT_PREFIX = "172.17.0.0"; private final Docker docker; + private final Environment environment; private final ProcessExecuter processExecuter; - private final ContainerEnvironmentResolver containerEnvironmentResolver; - private final List<String> configServerHostnames; - private final IPAddresses ipAddresses; - public DockerOperationsImpl(Docker docker, ProcessExecuter processExecuter, - ContainerEnvironmentResolver containerEnvironmentResolver, - List<String> configServerHostnames, IPAddresses ipAddresses) { + public DockerOperationsImpl(Docker docker, Environment environment, ProcessExecuter processExecuter) { this.docker = docker; + this.environment = environment; this.processExecuter = processExecuter; - this.containerEnvironmentResolver = containerEnvironmentResolver; - this.configServerHostnames = configServerHostnames; - this.ipAddresses = ipAddresses; } @Override @@ -66,11 +60,11 @@ public class DockerOperationsImpl implements DockerOperations { context.log(logger, "Creating container"); // IPv6 - Assume always valid - Inet6Address ipV6Address = ipAddresses.getIPv6Address(node.getHostname()).orElseThrow( + Inet6Address ipV6Address = environment.getIpAddresses().getIPv6Address(node.getHostname()).orElseThrow( () -> new RuntimeException("Unable to find a valid IPv6 address for " + node.getHostname() + ". Missing an AAAA DNS entry?")); - String configServers = String.join(",", configServerHostnames); + String configServers = String.join(",", environment.getConfigServerHostNames()); Docker.CreateContainerCommand command = docker.createContainerCommand( node.getWantedDockerImage().get(), @@ -79,7 +73,8 @@ public class DockerOperationsImpl implements DockerOperations { node.getHostname()) .withManagedBy(MANAGER_NAME) .withEnvironment("VESPA_CONFIGSERVERS", configServers) - .withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS", containerEnvironmentResolver.createSettings(node)) + .withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS", + environment.getContainerEnvironmentResolver().createSettings(environment, node)) .withUlimit("nofile", 262_144, 262_144) .withUlimit("nproc", 32_768, 409_600) .withUlimit("core", -1, -1) @@ -87,7 +82,7 @@ public class DockerOperationsImpl implements DockerOperations { .withAddCapability("SYS_ADMIN"); // Needed for perf - DockerNetworking networking = context.dockerNetworking(); + DockerNetworking networking = environment.getDockerNetworking(); command.withNetworkMode(networking.getDockerNetworkMode()); if (networking == DockerNetworking.NPT) { @@ -96,7 +91,7 @@ public class DockerOperationsImpl implements DockerOperations { command.withIpAddress(ipV6Local); // IPv4 - Only present for some containers - Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(node.getHostname()) + Optional<InetAddress> ipV4Local = environment.getIpAddresses().getIPv4Address(node.getHostname()) .map(ipV4Address -> { InetAddress ipV4Prefix = InetAddresses.forString(IPV4_NPT_PREFIX); return IPAddresses.prefixTranslate(ipV4Address, ipV4Prefix, 2); @@ -270,6 +265,7 @@ public class DockerOperationsImpl implements DockerOperations { // Paths unique to each container List<Path> paths = new ArrayList<>(Arrays.asList( Paths.get("/etc/yamas-agent"), + Paths.get("/etc/filebeat"), context.pathInNodeUnderVespaHome("logs/daemontools_y"), context.pathInNodeUnderVespaHome("logs/jdisc_core"), context.pathInNodeUnderVespaHome("logs/langdetect/"), diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 2b235ac9867..171e47d33c5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -1,12 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance; +import com.yahoo.collections.Pair; import com.yahoo.config.provision.NodeType; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; +import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -23,6 +28,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -31,7 +37,9 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.regex.Pattern; +import java.util.stream.Stream; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.olderThan; import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists; @@ -46,11 +54,16 @@ public class StorageMaintainer { .ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC); private final DockerOperations dockerOperations; + private final ProcessExecuter processExecuter; + private final Environment environment; private final CoredumpHandler coredumpHandler; private final Path archiveContainerStoragePath; - public StorageMaintainer(DockerOperations dockerOperations, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) { + public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter, + Environment environment, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) { this.dockerOperations = dockerOperations; + this.processExecuter = processExecuter; + this.environment = environment; this.coredumpHandler = coredumpHandler; this.archiveContainerStoragePath = archiveContainerStoragePath; } @@ -61,37 +74,37 @@ public class StorageMaintainer { // host-life Path hostLifeCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life"); SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath); - configs.add(annotatedCheck(context, node, hostLifeSchedule)); + configs.add(annotatedCheck(node, hostLifeSchedule)); // ntp Path ntpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ntp"); SecretAgentCheckConfig ntpSchedule = new SecretAgentCheckConfig("ntp", 60, ntpCheckPath); - configs.add(annotatedCheck(context, node, ntpSchedule)); + configs.add(annotatedCheck(node, ntpSchedule)); // coredumps (except for the done coredumps which is handled by the host) Path coredumpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps"); SecretAgentCheckConfig coredumpSchedule = new SecretAgentCheckConfig("system-coredumps-processing", 300, coredumpCheckPath, "--application", "system-coredumps-processing", "--lastmin", "129600", "--crit", "1", "--coredir", context.pathInNodeUnderVespaHome("var/crash/processing").toString()); - configs.add(annotatedCheck(context, node, coredumpSchedule)); + configs.add(annotatedCheck(node, coredumpSchedule)); // athenz certificate check Path athenzCertExpiryCheckPath = context.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs"); SecretAgentCheckConfig athenzCertExpirySchedule = new SecretAgentCheckConfig("athenz-certificate-expiry", 60, athenzCertExpiryCheckPath, "--threshold", "20") .withRunAsUser("root"); - configs.add(annotatedCheck(context, node, athenzCertExpirySchedule)); + configs.add(annotatedCheck(node, athenzCertExpirySchedule)); if (context.nodeType() != NodeType.config) { // vespa-health Path vespaHealthCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health"); SecretAgentCheckConfig vespaHealthSchedule = new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all"); - configs.add(annotatedCheck(context, node, vespaHealthSchedule)); + configs.add(annotatedCheck(node, vespaHealthSchedule)); // vespa Path vespaCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa"); SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all"); - configs.add(annotatedCheck(context, node, vespaSchedule)); + configs.add(annotatedCheck(node, vespaSchedule)); } if (context.nodeType() == NodeType.config) { @@ -99,14 +112,14 @@ public class StorageMaintainer { Path configServerCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ymonsb2"); SecretAgentCheckConfig configServerSchedule = new SecretAgentCheckConfig("configserver", 60, configServerCheckPath, "-zero", "configserver"); - configs.add(annotatedCheck(context, node, configServerSchedule)); + configs.add(annotatedCheck(node, configServerSchedule)); //zkbackupage Path zkbackupCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); SecretAgentCheckConfig zkbackupSchedule = new SecretAgentCheckConfig("zkbackupage", 300, zkbackupCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(), "-m", "150", "-a", "config-zkbackupage"); - configs.add(annotatedCheck(context, node, zkbackupSchedule)); + configs.add(annotatedCheck(node, zkbackupSchedule)); } if (context.nodeType() == NodeType.proxy) { @@ -115,13 +128,13 @@ public class StorageMaintainer { SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60, routingAgeCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(), "-m", "90", "-a", "routing-configage"); - configs.add(annotatedCheck(context, node, routingAgeSchedule)); + configs.add(annotatedCheck(node, routingAgeSchedule)); //ssl-check Path sslCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status"); SecretAgentCheckConfig sslSchedule = new SecretAgentCheckConfig("ssl-status", 300, sslCheckPath, "-e", "localhost", "-p", "4443", "-t", "30"); - configs.add(annotatedCheck(context, node, sslSchedule)); + configs.add(annotatedCheck(node, sslSchedule)); } // Write config and restart yamas-agent @@ -134,14 +147,14 @@ public class StorageMaintainer { dockerOperations.executeCommandInContainerAsRoot(context, "service", "yamas-agent", "restart"); } - private SecretAgentCheckConfig annotatedCheck(NodeAgentContext context, NodeSpec node, SecretAgentCheckConfig check) { + private SecretAgentCheckConfig annotatedCheck(NodeSpec node, SecretAgentCheckConfig check) { check.withTag("namespace", "Vespa") .withTag("role", SecretAgentCheckConfig.nodeTypeToRole(node.getNodeType())) .withTag("flavor", node.getFlavor()) .withTag("canonicalFlavor", node.getCanonicalFlavor()) .withTag("state", node.getState().toString()) - .withTag("zone", String.format("%s.%s", context.zoneId().environment().value(), context.zoneId().regionName().value())); - node.getParentHostname().ifPresent(parent -> check.withTag("parentHostname", parent)); + .withTag("zone", environment.getZone()) + .withTag("parentHostname", environment.getParentHostHostname()); node.getOwner().ifPresent(owner -> check .withTag("tenantName", owner.getTenant()) .withTag("app", owner.getApplication() + "." + owner.getInstance()) @@ -156,6 +169,20 @@ public class StorageMaintainer { return check; } + public void writeFilebeatConfig(NodeAgentContext context, NodeSpec node) { + try { + FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment); + Optional<String> config = filebeatConfigProvider.getConfig(context, node); + if (!config.isPresent()) return; + + Path filebeatPath = context.pathOnHostFromPathInNode("/etc/filebeat/filebeat.yml"); + Files.write(filebeatPath, config.get().getBytes()); + context.log(logger, "Wrote filebeat config"); + } catch (Throwable t) { + context.log(logger, LogLevel.ERROR, "Failed writing filebeat config", t); + } + } + public Optional<Long> getDiskUsageFor(NodeAgentContext context) { Path containerDir = context.pathOnHostFromPathInNode("/"); try { @@ -220,20 +247,20 @@ public class StorageMaintainer { /** Checks if container has any new coredumps, reports and archives them if so */ public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node, Optional<Container> container) { - final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(context, node, container); + final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(node, container); coredumpHandler.converge(context, nodeAttributes); } - private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, NodeSpec node, Optional<Container> container) { + private Map<String, Object> getCoredumpNodeAttributes(NodeSpec node, Optional<Container> container) { Map<String, Object> attributes = new HashMap<>(); attributes.put("hostname", node.getHostname()); - attributes.put("region", context.zoneId().regionName()); - attributes.put("environment", context.zoneId().environment()); + attributes.put("parent_hostname", environment.getParentHostHostname()); + attributes.put("region", environment.getRegion()); + attributes.put("environment", environment.getEnvironment()); attributes.put("flavor", node.getFlavor()); attributes.put("kernel_version", System.getProperty("os.version")); container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString())); - node.getParentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); node.getVespaVersion().ifPresent(version -> attributes.put("vespa_version", version)); node.getOwner().ifPresent(owner -> { attributes.put("tenant", owner.getTenant()); @@ -257,4 +284,56 @@ public class StorageMaintainer { new UnixPath(containerLogsOnHost).moveIfExists(containerLogsInArchiveDir); new UnixPath(context.pathOnHostFromPathInNode("/")).deleteRecursively(); } + + /** + * Runs node-maintainer's SpecVerifier and returns its output + * @param node Node specification containing the excepted values we want to verify against + * @return new combined hardware divergence + * @throws RuntimeException if exit code != 0 + */ + public String getHardwareDivergence(NodeSpec node) { + List<String> arguments = new ArrayList<>(Arrays.asList("specification", + "--disk", Double.toString(node.getMinDiskAvailableGb()), + "--memory", Double.toString(node.getMinMainMemoryAvailableGb()), + "--cpu_cores", Double.toString(node.getMinCpuCores()), + "--is_ssd", Boolean.toString(node.isFastDisk()), + "--bandwidth", Double.toString(node.getBandwidth()), + "--ips", String.join(",", node.getIpAddresses()))); + + if (environment.getDockerNetworking() == DockerNetworking.HOST_NETWORK) { + arguments.add("--skip-reverse-lookup"); + } + + node.getHardwareDivergence().ifPresent(hardwareDivergence -> { + arguments.add("--divergence"); + arguments.add(hardwareDivergence); + }); + + return executeMaintainer("com.yahoo.vespa.hosted.node.verification.Main", arguments.toArray(new String[0])); + } + + + private String executeMaintainer(String mainClass, String... args) { + String[] command = Stream.concat( + Stream.of("sudo", + "VESPA_HOSTNAME=" + getDefaults().vespaHostname(), + "VESPA_HOME=" + getDefaults().vespaHome(), + getDefaults().underVespaHome("libexec/vespa/node-admin/maintenance.sh"), + mainClass), + Stream.of(args)) + .toArray(String[]::new); + + try { + Pair<Integer, String> result = processExecuter.exec(command); + + if (result.getFirst() != 0) { + throw new RuntimeException( + String.format("Maintainer failed to execute command: %s, Exit code: %d, Stdout/stderr: %s", + Arrays.toString(command), result.getFirst(), result.getSecond())); + } + return result.getSecond().trim(); + } catch (IOException e) { + throw new RuntimeException("Failed to execute maintainer", e); + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index a733a52ea10..88fb5dcb90f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -2,10 +2,11 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.acl; import com.google.common.net.InetAddresses; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.dockerapi.Container; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; @@ -30,21 +31,23 @@ import java.util.stream.Collectors; * @author mpolden * @author smorgrav */ -public class AclMaintainer { +public class AclMaintainer implements Runnable { private static final PrefixLogger log = PrefixLogger.getNodeAdminLogger(AclMaintainer.class); private final DockerOperations dockerOperations; private final NodeRepository nodeRepository; private final IPAddresses ipAddresses; - private final String hostHostname; + private final String nodeAdminHostname; + private final Environment environment; public AclMaintainer(DockerOperations dockerOperations, NodeRepository nodeRepository, - HostName hostHostname, IPAddresses ipAddresses) { + String nodeAdminHostname, IPAddresses ipAddresses, Environment environment) { this.dockerOperations = dockerOperations; this.nodeRepository = nodeRepository; this.ipAddresses = ipAddresses; - this.hostHostname = hostHostname.value(); + this.nodeAdminHostname = nodeAdminHostname; + this.environment = environment; } private void applyRedirect(Container container, InetAddress address) { @@ -65,18 +68,21 @@ public class AclMaintainer { } private synchronized void configureAcls() { + if (environment.getDockerNetworking() != DockerNetworking.NPT) return; + log.info("Configuring ACLs"); // Needed to potentially nail down when ACL maintainer stopped working Map<String, Container> runningContainers = dockerOperations .getAllManagedContainers().stream() .filter(container -> container.state.isRunning()) .collect(Collectors.toMap(container -> container.hostname, container -> container)); - nodeRepository.getAcls(hostHostname).entrySet().stream() + nodeRepository.getAcls(nodeAdminHostname).entrySet().stream() .filter(entry -> runningContainers.containsKey(entry.getKey())) .forEach(entry -> apply(runningContainers.get(entry.getKey()), entry.getValue())); } - public void converge() { + @Override + public void run() { try { configureAcls(); } catch (Throwable t) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index b1c1a99a90a..d436e214266 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; @@ -18,7 +18,6 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -40,7 +39,7 @@ public class NodeAdminImpl implements NodeAdmin { Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler")); private final Function<String, NodeAgent> nodeAgentFactory; - private final Optional<AclMaintainer> aclMaintainer; + private final Runnable aclMaintainer; private final Clock clock; private boolean previousWantFrozen; @@ -52,8 +51,16 @@ public class NodeAdminImpl implements NodeAdmin { private final GaugeWrapper numberOfContainersInLoadImageState; private final CounterWrapper numberOfUnhandledExceptionsInNodeAgent; + public NodeAdminImpl(DockerOperations dockerOperations, + Function<String, NodeAgent> nodeAgentFactory, + Runnable aclMaintainer, + MetricReceiverWrapper metricReceiver, + Clock clock) { + this(nodeAgentFactory, aclMaintainer, metricReceiver, clock); + } + public NodeAdminImpl(Function<String, NodeAgent> nodeAgentFactory, - Optional<AclMaintainer> aclMaintainer, + Runnable aclMaintainer, MetricReceiverWrapper metricReceiver, Clock clock) { this.nodeAgentFactory = nodeAgentFactory; @@ -168,12 +175,10 @@ public class NodeAdminImpl implements NodeAdmin { } }, 10, 55, TimeUnit.SECONDS); - aclMaintainer.ifPresent(maintainer -> { - int delay = 120; // WARNING: Reducing this will increase the load on config servers. - aclScheduler.scheduleWithFixedDelay(() -> { - if (!isFrozen()) maintainer.converge(); - }, 30, delay, TimeUnit.SECONDS); - }); + int delay = 120; // WARNING: Reducing this will increase the load on config servers. + aclScheduler.scheduleWithFixedDelay(() -> { + if (!isFrozen()) aclMaintainer.run(); + }, 30, delay, TimeUnit.SECONDS); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java index 296745c8e37..5b0a3c7771f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; -import com.yahoo.config.provision.HostName; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.concurrent.classlock.ClassLock; +import com.yahoo.concurrent.classlock.ClassLocking; +import com.yahoo.concurrent.classlock.LockInterruptException; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; @@ -19,7 +24,11 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -45,34 +54,55 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { private RuntimeException lastConvergenceException; private final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); + private final ScheduledExecutorService specVerifierScheduler = + Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("specverifier")); private final Thread loopThread; private final NodeRepository nodeRepository; private final Orchestrator orchestrator; private final NodeAdmin nodeAdmin; private final Clock clock; - private final String hostHostname; + private final String dockerHostHostName; private final Duration nodeAdminConvergeStateInterval; + private final Optional<ClassLocking> classLocking; + private Optional<ClassLock> classLock = Optional.empty(); private Instant lastTick; public NodeAdminStateUpdaterImpl( NodeRepository nodeRepository, Orchestrator orchestrator, + StorageMaintainer storageMaintainer, NodeAdmin nodeAdmin, - HostName hostHostname, + String dockerHostHostName, Clock clock, - Duration nodeAdminConvergeStateInterval) { + Duration nodeAdminConvergeStateInterval, + Optional<ClassLocking> classLocking) { + log.info(objectToString() + ": Creating object"); this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; this.nodeAdmin = nodeAdmin; - this.hostHostname = hostHostname.value(); + this.dockerHostHostName = dockerHostHostName; this.clock = clock; this.nodeAdminConvergeStateInterval = nodeAdminConvergeStateInterval; + this.classLocking = classLocking; this.lastTick = clock.instant(); this.loopThread = new Thread(() -> { + if (classLocking.isPresent()) { + log.info(objectToString() + ": Acquiring lock"); + try { + classLock = Optional.of(classLocking.get().lockWhile(NodeAdminStateUpdater.class, () -> !terminated.get())); + } catch (LockInterruptException e) { + classLock = Optional.empty(); + return; + } + } + + log.info(objectToString() + ": Starting threads and schedulers"); nodeAdmin.start(); + specVerifierScheduler.scheduleWithFixedDelay(() -> + updateHardwareDivergence(storageMaintainer), 5, 60, TimeUnit.MINUTES); while (! terminated.get()) { tick(); @@ -81,11 +111,15 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { this.loopThread.setName("tick-NodeAdminStateUpdater"); } + private String objectToString() { + return this.getClass().getSimpleName() + "@" + Integer.toString(System.identityHashCode(this)); + } + @Override public Map<String, Object> getDebugPage() { Map<String, Object> debug = new LinkedHashMap<>(); synchronized (monitor) { - debug.put("hostHostname", hostHostname); + debug.put("dockerHostHostName", dockerHostHostName); debug.put("wantedState", wantedState); debug.put("currentState", currentState); debug.put("NodeAdmin", nodeAdmin.debugInfo()); @@ -93,6 +127,25 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { return debug; } + private void updateHardwareDivergence(StorageMaintainer maintainer) { + if (currentState != RESUMED) return; + + try { + NodeSpec node = nodeRepository.getNode(dockerHostHostName); + if (node.getState() == Node.State.parked) return; + String hardwareDivergence = maintainer.getHardwareDivergence(node); + + // Only update hardware divergence if there is a change. + if (!node.getHardwareDivergence().orElse("null").equals(hardwareDivergence)) { + NodeAttributes nodeAttributes = new NodeAttributes().withHardwareDivergence(hardwareDivergence); + log.info("Updating hardware divergence to " + hardwareDivergence); + nodeRepository.updateNodeAttributes(dockerHostHostName, nodeAttributes); + } + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to report hardware divergence", e); + } + } + @Override public void setResumeStateAndCheckIfResumed(State wantedState) { synchronized (monitor) { @@ -186,13 +239,13 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { throw new ConvergenceException("NodeAdmin is not yet " + (wantFrozen ? "frozen" : "unfrozen")); } - boolean hostIsActiveInNR = nodeRepository.getNode(hostHostname).getState() == Node.State.active; + boolean hostIsActiveInNR = nodeRepository.getNode(dockerHostHostName).getState() == Node.State.active; switch (wantedState) { case RESUMED: - if (hostIsActiveInNR) orchestrator.resume(hostHostname); + if (hostIsActiveInNR) orchestrator.resume(dockerHostHostName); break; case SUSPENDED_NODE_ADMIN: - if (hostIsActiveInNR) orchestrator.suspend(hostHostname); + if (hostIsActiveInNR) orchestrator.suspend(dockerHostHostName); break; case SUSPENDED: // Fetch active nodes from node repo before suspending nodes. @@ -204,9 +257,9 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { List<String> nodesInActiveState = getNodesInActiveState(); List<String> nodesToSuspend = new ArrayList<>(nodesInActiveState); - if (hostIsActiveInNR) nodesToSuspend.add(hostHostname); + if (hostIsActiveInNR) nodesToSuspend.add(dockerHostHostName); if (!nodesToSuspend.isEmpty()) { - orchestrator.suspend(hostHostname, nodesToSuspend); + orchestrator.suspend(dockerHostHostName, nodesToSuspend); log.info("Orchestrator allows suspension of " + nodesToSuspend); } @@ -228,7 +281,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { private void fetchContainersToRunFromNodeRepository() { try { - final List<NodeSpec> containersToRun = nodeRepository.getNodes(hostHostname); + final List<NodeSpec> containersToRun = nodeRepository.getNodes(dockerHostHostName); nodeAdmin.refreshContainersToRun(containersToRun); } catch (Exception e) { log.log(LogLevel.WARNING, "Failed to update which containers should be running", e); @@ -236,7 +289,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { } private List<String> getNodesInActiveState() { - return nodeRepository.getNodes(hostHostname) + return nodeRepository.getNodes(dockerHostHostName) .stream() .filter(node -> node.getState() == Node.State.active) .map(NodeSpec::getHostname) @@ -248,22 +301,33 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { } public void stop() { + log.info(objectToString() + ": Stop called"); if (!terminated.compareAndSet(false, true)) { throw new RuntimeException("Can not re-stop a node agent."); } + classLocking.ifPresent(ClassLocking::interrupt); + // First we need to stop NodeAdminStateUpdaterImpl thread to make sure no new NodeAgents are spawned signalWorkToBeDone(); + specVerifierScheduler.shutdown(); do { try { loopThread.join(); + specVerifierScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e1) { log.info("Interrupted while waiting for NodeAdminStateUpdater thread and specVerfierScheduler to shutdown"); } - } while (loopThread.isAlive()); + } while (loopThread.isAlive() || !specVerifierScheduler.isTerminated()); // Finally, stop NodeAdmin and all the NodeAgents nodeAdmin.stop(); + + classLock.ifPresent(lock -> { + log.info(objectToString() + ": Releasing lock"); + lock.close(); + }); + log.info(objectToString() + ": Stop complete"); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index 2025d422331..f65f371ff67 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.component.ZoneId; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import java.nio.file.Path; import java.nio.file.Paths; @@ -21,10 +19,6 @@ public interface NodeAgentContext extends TaskContext { AthenzService identity(); - DockerNetworking dockerNetworking(); - - ZoneId zoneId(); - /** * This method is the inverse of {@link #pathInNodeFromPathOnHost(Path)}} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index d1b5ec77c6a..d3c8b145488 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -1,20 +1,13 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.node.admin.component.ZoneId; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import java.nio.file.FileSystem; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Objects; -import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -29,20 +22,15 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final HostName hostName; private final NodeType nodeType; private final AthenzService identity; - private final DockerNetworking dockerNetworking; - private final ZoneId zoneId; private final Path pathToNodeRootOnHost; private final Path pathToVespaHome; public NodeAgentContextImpl(String hostname, NodeType nodeType, AthenzService identity, - DockerNetworking dockerNetworking, ZoneId zoneId, Path pathToContainerStorage, Path pathToVespaHome) { this.hostName = HostName.from(Objects.requireNonNull(hostname)); this.containerName = ContainerName.fromHostname(hostname); this.nodeType = Objects.requireNonNull(nodeType); this.identity = Objects.requireNonNull(identity); - this.dockerNetworking = Objects.requireNonNull(dockerNetworking); - this.zoneId = Objects.requireNonNull(zoneId); this.pathToNodeRootOnHost = Objects.requireNonNull(pathToContainerStorage).resolve(containerName.asString()); this.pathToVespaHome = Objects.requireNonNull(pathToVespaHome); this.logPrefix = containerName.asString() + ": "; @@ -69,16 +57,6 @@ public class NodeAgentContextImpl implements NodeAgentContext { } @Override - public DockerNetworking dockerNetworking() { - return dockerNetworking; - } - - @Override - public ZoneId zoneId() { - return zoneId; - } - - @Override public Path pathOnHostFromPathInNode(Path pathInNode) { if (! pathInNode.isAbsolute()) throw new IllegalArgumentException("Expected an absolute path in the container, got: " + pathInNode); @@ -119,66 +97,4 @@ public class NodeAgentContextImpl implements NodeAgentContext { public void log(Logger logger, Level level, String message, Throwable throwable) { logger.log(level, logPrefix + message, throwable); } - - - /** For testing only! */ - public static class Builder { - private final String hostname; - private NodeType nodeType; - private AthenzService identity; - private DockerNetworking dockerNetworking; - private ZoneId zoneId; - private Path pathToContainerStorage; - private Path pathToVespaHome; - - public Builder(String hostname) { - this.hostname = hostname; - } - - public Builder nodeType(NodeType nodeType) { - this.nodeType = nodeType; - return this; - } - - public Builder identity(AthenzService identity) { - this.identity = identity; - return this; - } - - public Builder dockerNetworking(DockerNetworking dockerNetworking) { - this.dockerNetworking = dockerNetworking; - return this; - } - - public Builder zoneId(ZoneId zoneId) { - this.zoneId = zoneId; - return this; - } - - public Builder pathToContainerStorage(Path pathToContainerStorage) { - this.pathToContainerStorage = pathToContainerStorage; - return this; - } - - public Builder pathToVespaHome(Path pathToVespaHome) { - this.pathToVespaHome = pathToVespaHome; - return this; - } - - public Builder fileSystem(FileSystem fileSystem) { - return pathToContainerStorage(fileSystem.getPath("/home/docker")); - } - - public NodeAgentContextImpl build() { - return new NodeAgentContextImpl( - hostname, - Optional.ofNullable(nodeType).orElse(NodeType.tenant), - Optional.ofNullable(identity).orElseGet(() -> new AthenzService("domain", "service")), - Optional.ofNullable(dockerNetworking).orElse(DockerNetworking.HOST_NETWORK), - Optional.ofNullable(zoneId).orElseGet(() -> new ZoneId(SystemName.dev, Environment.dev, RegionName.defaultName())), - Optional.ofNullable(pathToContainerStorage).orElseGet(() -> Paths.get("/home/docker")), - Optional.ofNullable(pathToVespaHome).orElseGet(() -> Paths.get("/opt/vespa")) - ); - } - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 395b4d458a2..ef2c6ad2800 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -22,7 +22,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.provision.Node; @@ -72,10 +72,11 @@ public class NodeAgentImpl implements NodeAgent { private final Orchestrator orchestrator; private final DockerOperations dockerOperations; private final StorageMaintainer storageMaintainer; + private final Runnable aclMaintainer; + private final Environment environment; private final Clock clock; private final Duration timeBetweenEachConverge; private final Optional<AthenzCredentialsMaintainer> athenzCredentialsMaintainer; - private final Optional<AclMaintainer> aclMaintainer; private int numberOfUnhandledException = 0; private Instant lastConverge; @@ -115,20 +116,22 @@ public class NodeAgentImpl implements NodeAgent { final Orchestrator orchestrator, final DockerOperations dockerOperations, final StorageMaintainer storageMaintainer, + final Runnable aclMaintainer, + final Environment environment, final Clock clock, final Duration timeBetweenEachConverge, - final Optional<AthenzCredentialsMaintainer> athenzCredentialsMaintainer, - final Optional<AclMaintainer> aclMaintainer) { + final Optional<AthenzCredentialsMaintainer> athenzCredentialsMaintainer) { this.context = context; this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; this.dockerOperations = dockerOperations; this.storageMaintainer = storageMaintainer; + this.aclMaintainer = aclMaintainer; + this.environment = environment; this.clock = clock; this.timeBetweenEachConverge = timeBetweenEachConverge; this.lastConverge = clock.instant(); this.athenzCredentialsMaintainer = athenzCredentialsMaintainer; - this.aclMaintainer = aclMaintainer; this.loopThread = new Thread(() -> { try { @@ -225,6 +228,7 @@ public class NodeAgentImpl implements NodeAgent { if (!hasResumedNode) { if (!currentFilebeatRestarter.isPresent()) { storageMaintainer.writeMetricsConfig(context, node); + storageMaintainer.writeFilebeatConfig(context, node); currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay( () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS)); } @@ -505,7 +509,7 @@ public class NodeAgentImpl implements NodeAgent { containerState = STARTING; startContainer(node); containerState = UNKNOWN; - aclMaintainer.ifPresent(AclMaintainer::converge); + aclMaintainer.run(); } verifyHealth(node); @@ -602,8 +606,8 @@ public class NodeAgentImpl implements NodeAgent { Dimensions.Builder dimensionsBuilder = new Dimensions.Builder() .add("host", context.hostname().value()) .add("role", "tenants") - .add("state", node.getState().toString()); - node.getParentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent)); + .add("state", node.getState().toString()) + .add("parentHostname", environment.getParentHostHostname()); node.getAllowedToBeDown().ifPresent(allowed -> dimensionsBuilder.add("orchestratorState", allowed ? "ALLOWED_TO_BE_DOWN" : "NO_REMARKS")); Dimensions dimensions = dimensionsBuilder.build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 2a4bf6bf488..760326f061e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -9,19 +9,17 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.node.admin.component.ContainerEnvironmentResolver; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest; import org.junit.Test; import org.mockito.InOrder; import java.io.IOException; import java.net.InetAddress; import java.nio.file.Paths; -import java.util.Collections; import java.util.Optional; import java.util.OptionalLong; @@ -38,16 +36,22 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class DockerOperationsImplTest { + private final Environment environment = new Environment.Builder() + .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) + .region("us-east-1") + .environment("prod") + .system("main") + .cloud("mycloud") + .dockerNetworking(DockerNetworking.HOST_NETWORK) + .build(); private final Docker docker = mock(Docker.class); private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); - private final ContainerEnvironmentResolver containerEnvironmentResolver = node -> ""; - private final IPAddresses ipAddresses = new IPAddressesMock(); - private final DockerOperationsImpl dockerOperations = new DockerOperationsImpl( - docker, processExecuter, containerEnvironmentResolver, Collections.emptyList(), ipAddresses); + private final DockerOperationsImpl dockerOperations + = new DockerOperationsImpl(docker, environment, processExecuter); @Test public void processResultFromNodeProgramWhenSuccess() { - final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); + final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("container-123.domain.tld"); final ProcessResult actualResult = new ProcessResult(0, "output", "errors"); when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg())) @@ -68,7 +72,7 @@ public class DockerOperationsImplTest { @Test(expected = RuntimeException.class) public void processResultFromNodeProgramWhenNonZeroExitCode() { - final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); + final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("container-123.domain.tld"); final ProcessResult actualResult = new ProcessResult(3, "output", "errors"); when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg())) @@ -101,6 +105,10 @@ public class DockerOperationsImplTest { InetAddress ipV6Local = InetAddresses.forString("::1"); InetAddress ipV4Local = InetAddresses.forString("127.0.0.1"); + DockerOperationsImpl dockerOperations = new DockerOperationsImpl( + docker, + environment, + processExecuter); dockerOperations.addEtcHosts(containerData, hostname, Optional.empty(), ipV6Local); verify(containerData, times(1)).addFile( diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index aa40d5d805b..bca9dbc44fa 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -2,20 +2,25 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.collections.Pair; -import com.yahoo.config.provision.HostName; +import com.yahoo.concurrent.classlock.ClassLocking; import com.yahoo.config.provision.NodeType; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; +import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.test.file.TestFileSystem; @@ -25,7 +30,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; -import java.util.Collections; import java.util.Optional; import java.util.function.Function; @@ -43,7 +47,7 @@ public class DockerTester implements AutoCloseable { private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofMillis(10); private static final Path PATH_TO_VESPA_HOME = Paths.get("/opt/vespa"); static final String NODE_PROGRAM = PATH_TO_VESPA_HOME.resolve("bin/vespa-nodectl").toString(); - static final HostName HOST_HOSTNAME = HostName.from("host.test.yahoo.com"); + static final String DOCKER_HOST_HOSTNAME = "host.test.yahoo.com"; final CallOrderVerifier callOrderVerifier = new CallOrderVerifier(); final Docker dockerMock = new DockerMock(callOrderVerifier); @@ -56,15 +60,26 @@ public class DockerTester implements AutoCloseable { DockerTester() { IPAddressesMock ipAddresses = new IPAddressesMock(); - ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1"); - ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::"); + ipAddresses.addAddress(DOCKER_HOST_HOSTNAME, "1.1.1.1"); + ipAddresses.addAddress(DOCKER_HOST_HOSTNAME, "f000::"); for (int i = 1; i < 4; i++) ipAddresses.addAddress("host" + i + ".test.yahoo.com", "f000::" + i); ProcessExecuter processExecuter = mock(ProcessExecuter.class); uncheck(() -> when(processExecuter.exec(any(String[].class))).thenReturn(new Pair<>(0, ""))); + Environment environment = new Environment.Builder() + .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) + .ipAddresses(ipAddresses) + .region("us-east-1") + .environment("prod") + .system("main") + .cloud("mycloud") + .pathResolver(new PathResolver(PATH_TO_VESPA_HOME, Paths.get("/tmp"), Paths.get("/tmp"))) + .dockerNetworking(DockerNetworking.HOST_NETWORK) + .build(); + NodeSpec hostSpec = new NodeSpec.Builder() - .hostname(HOST_HOSTNAME.value()) + .hostname(DOCKER_HOST_HOSTNAME) .state(Node.State.active) .nodeType(NodeType.host) .flavor("default") @@ -74,23 +89,26 @@ public class DockerTester implements AutoCloseable { nodeRepositoryMock.updateNodeRepositoryNode(hostSpec); Clock clock = Clock.systemUTC(); - DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, processExecuter, node -> "", Collections.emptyList(), ipAddresses); - StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, callOrderVerifier); + DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, processExecuter); + StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, null, environment, callOrderVerifier); + AclMaintainer aclMaintainer = mock(AclMaintainer.class); + MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl( - new NodeAgentContextImpl.Builder(hostName).fileSystem(fileSystem).build(), nodeRepositoryMock, - orchestratorMock, dockerOperations, storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.empty(), Optional.empty()); - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, Optional.empty(), mr, Clock.systemUTC()); - nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, - nodeAdmin, HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL); + NodeAgentContextImplTest.nodeAgentFromHostname(fileSystem, hostName), nodeRepositoryMock, + orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL, Optional.empty()); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, aclMaintainer, mr, Clock.systemUTC()); + nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer, + nodeAdmin, DOCKER_HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, + Optional.of(new ClassLocking())); nodeAdminStateUpdater.start(); } /** Adds a node to node-repository mock that is running on this host */ void addChildNodeRepositoryNode(NodeSpec nodeSpec) { nodeRepositoryMock.updateNodeRepositoryNode(new NodeSpec.Builder(nodeSpec) - .parentHostname(HOST_HOSTNAME.value()) + .parentHostname(DOCKER_HOST_HOSTNAME) .build()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index 298c185266b..e91787ca540 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -1,11 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; +import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import java.util.Optional; @@ -15,8 +17,8 @@ import java.util.Optional; public class StorageMaintainerMock extends StorageMaintainer { private final CallOrderVerifier callOrderVerifier; - public StorageMaintainerMock(DockerOperations dockerOperations, CallOrderVerifier callOrderVerifier) { - super(dockerOperations, null, null); + public StorageMaintainerMock(DockerOperations dockerOperations, ProcessExecuter processExecuter, Environment environment, CallOrderVerifier callOrderVerifier) { + super(dockerOperations, processExecuter, environment, null, null); this.callOrderVerifier = callOrderVerifier; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java index d4fadabe695..f7e10c3aa13 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; @@ -32,7 +32,7 @@ public class FilebeatConfigProviderTest { private static final String region = "us-north-1"; private static final String system = "main"; private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2"); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("node-123.hostname.tld").build(); + private final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("node-123.hostname.tld"); @Test public void it_replaces_all_fields_correctly() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 42b161b8e2d..7c10e703bb5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -2,9 +2,14 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.google.common.collect.ImmutableSet; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; +import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Rule; @@ -30,14 +35,24 @@ import static org.mockito.Mockito.mock; * @author dybis */ public class StorageMaintainerTest { + private final Environment environment = new Environment.Builder() + .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) + .region("us-east-1") + .environment("prod") + .system("main") + .cloud("mycloud") + .pathResolver(new PathResolver()) + .dockerNetworking(DockerNetworking.HOST_NETWORK) + .build(); private final DockerOperations docker = mock(DockerOperations.class); + private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); @Rule public TemporaryFolder folder = new TemporaryFolder(); @Test public void testDiskUsed() throws IOException, InterruptedException { - StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, null); + StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, environment, null, null); int writeSize = 10000; Files.write(folder.newFile().toPath(), new byte[writeSize]); @@ -48,7 +63,7 @@ public class StorageMaintainerTest { @Test public void testNonExistingDiskUsed() throws IOException, InterruptedException { - StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, null); + StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, environment, null, null); long usedBytes = storageMaintainer.getDiskUsedInBytes(folder.getRoot().toPath().resolve("doesn't exist")); assertEquals(0L, usedBytes); } @@ -73,7 +88,7 @@ public class StorageMaintainerTest { // Archive container-1 - StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, pathToArchiveDir); + StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, environment, null, pathToArchiveDir); storageMaintainer.archiveNodeStorage(context1); // container-1 should be gone from container-storage @@ -97,8 +112,7 @@ public class StorageMaintainerTest { } private NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { - NodeAgentContext context = new NodeAgentContextImpl.Builder(containerName + ".domain.tld") - .fileSystem(fileSystem).build(); + NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname(fileSystem, containerName + ".domain.tld"); Path containerVespaHomeOnHost = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("")); Files.createDirectories(context.pathOnHostFromPathInNode("/etc/something")); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index a270585bcaa..5b882461937 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -1,13 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.acl; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; @@ -31,18 +32,21 @@ import static org.mockito.Mockito.when; public class AclMaintainerTest { - private final HostName hostHostname = HostName.from("node-admin.region-1.yahoo.com"); + private static final String NODE_ADMIN_HOSTNAME = "node-admin.region-1.yahoo.com"; + private final IPAddressesMock ipAddresses = new IPAddressesMock(); private final DockerOperations dockerOperations = mock(DockerOperations.class); private final NodeRepository nodeRepository = mock(NodeRepository.class); private final Map<String, Container> containers = new HashMap<>(); private final List<Container> containerList = new ArrayList<>(); + private final Environment env = mock(Environment.class); private final AclMaintainer aclMaintainer = - new AclMaintainer(dockerOperations, nodeRepository, hostHostname, ipAddresses); + new AclMaintainer(dockerOperations, nodeRepository, NODE_ADMIN_HOSTNAME, ipAddresses, env); @Before public void before() { when(dockerOperations.getAllManagedContainers()).thenReturn(containerList); + when(env.getDockerNetworking()).thenReturn(DockerNetworking.NPT); } @Test @@ -50,14 +54,14 @@ public class AclMaintainerTest { Container container = addContainer("container1", "container1.host.com", Container.State.RUNNING); Map<String, Acl> acls = makeAcl(container.hostname, "4321", "2001::1"); - when(nodeRepository.getAcls(hostHostname.value())).thenReturn(acls); + when(nodeRepository.getAcls(NODE_ADMIN_HOSTNAME)).thenReturn(acls); whenListRules(container.name, "filter", IPVersion.IPv6, ""); whenListRules(container.name, "filter", IPVersion.IPv4, ""); whenListRules(container.name, "nat", IPVersion.IPv4, ""); whenListRules(container.name, "nat", IPVersion.IPv6, ""); - aclMaintainer.converge(); + aclMaintainer.run(); verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg()); //we don;t have a ip4 address for the container so no redirect either verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg()); @@ -68,14 +72,14 @@ public class AclMaintainerTest { Container container = addContainer("container1", "container1.host.com", Container.State.RUNNING); Map<String, Acl> acls = makeAcl(container.hostname, "4321", "2001::1"); - when(nodeRepository.getAcls(hostHostname.value())).thenReturn(acls); + when(nodeRepository.getAcls(NODE_ADMIN_HOSTNAME)).thenReturn(acls); whenListRules(container.name, "filter", IPVersion.IPv6, ""); whenListRules(container.name, "filter", IPVersion.IPv4, ""); whenListRules(container.name, "nat", IPVersion.IPv4, ""); whenListRules(container.name, "nat", IPVersion.IPv6, ""); - aclMaintainer.converge(); + aclMaintainer.run(); verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg()); //we don;t have a ip4 address for the container so no redirect either verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg()); @@ -86,9 +90,9 @@ public class AclMaintainerTest { Container container = addContainer("container1", "container1.host.com", Container.State.EXITED); Map<String, Acl> acls = makeAcl(container.hostname, "4321", "2001::1"); - when(nodeRepository.getAcls(hostHostname.value())).thenReturn(acls); + when(nodeRepository.getAcls(NODE_ADMIN_HOSTNAME)).thenReturn(acls); - aclMaintainer.converge(); + aclMaintainer.run(); verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), anyVararg()); } @@ -98,7 +102,7 @@ public class AclMaintainerTest { Container container = addContainer("container1", "container1.host.com", Container.State.RUNNING); Map<String, Acl> acls = makeAcl(container.hostname, "4321,2345,22", "2001::1", "fd01:1234::4321"); - when(nodeRepository.getAcls(hostHostname.value())).thenReturn(acls); + when(nodeRepository.getAcls(NODE_ADMIN_HOSTNAME)).thenReturn(acls); String IPV6 = "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -121,7 +125,7 @@ public class AclMaintainerTest { whenListRules(container.name, "filter", IPVersion.IPv4, ""); //IPv4 will then differ from wanted whenListRules(container.name, "nat", IPVersion.IPv6, NATv6); - aclMaintainer.converge(); + aclMaintainer.run(); verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg()); verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg()); @@ -132,7 +136,7 @@ public class AclMaintainerTest { Container container = addContainer("container1", "container1.host.com", Container.State.RUNNING); Map<String, Acl> acls = makeAcl(container.hostname, "22,4443,2222", "2001::1", "192.64.13.2"); - when(nodeRepository.getAcls(hostHostname.value())).thenReturn(acls); + when(nodeRepository.getAcls(NODE_ADMIN_HOSTNAME)).thenReturn(acls); String IPV4_FILTER = "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -164,7 +168,7 @@ public class AclMaintainerTest { whenListRules(container.name, "nat", IPVersion.IPv6, IPV6_NAT); whenListRules(container.name, "filter", IPVersion.IPv4, IPV4_FILTER); - aclMaintainer.converge(); + aclMaintainer.run(); verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("ip6tables-restore"), anyVararg()); verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("iptables-restore"), anyVararg()); @@ -175,7 +179,7 @@ public class AclMaintainerTest { public void rollback_is_attempted_when_applying_acl_fail() { Container container = addContainer("container1", "container1.host.com", Container.State.RUNNING); Map<String, Acl> acls = makeAcl(container.hostname, "4321", "2001::1"); - when(nodeRepository.getAcls(hostHostname.value())).thenReturn(acls); + when(nodeRepository.getAcls(NODE_ADMIN_HOSTNAME)).thenReturn(acls); String IPV6_NAT = "-P PREROUTING ACCEPT\n" + "-P INPUT ACCEPT\n" + @@ -195,7 +199,7 @@ public class AclMaintainerTest { eq(container.name), eq("iptables-restore"), anyVararg())).thenThrow(new RuntimeException("iptables restore failed")); - aclMaintainer.converge(); + aclMaintainer.run(); verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables"), eq("-F"), eq("-t"), eq("filter")); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index d809d9cbf96..0fed9433432 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest; import org.junit.Test; import java.nio.file.Path; @@ -27,7 +27,7 @@ public class CoreCollectorTest { private final String GDB_PATH = "/my/path/to/gdb"; private final DockerOperations docker = mock(DockerOperations.class); private final CoreCollector coreCollector = new CoreCollector(docker, Paths.get(GDB_PATH)); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); + private final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname("container-123.domain.tld"); private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234"); private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index ca8435c9edd..d567cac9a70 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.google.common.collect.ImmutableMap; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImplTest; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; @@ -47,8 +47,7 @@ import static org.mockito.Mockito.when; public class CoredumpHandlerTest { private final FileSystem fileSystem = TestFileSystem.create(); private final Path donePath = fileSystem.getPath("/home/docker/dumps"); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld") - .fileSystem(fileSystem).build(); + private final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname(fileSystem, "container-123.domain.tld"); private final Path crashPathInContainer = Paths.get("/var/crash"); private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index bb229a10f59..2431433b17f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -14,7 +14,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -38,9 +37,10 @@ public class NodeAdminImplTest { // Trick to allow mocking of typed interface without casts/warnings. private interface NodeAgentFactory extends Function<String, NodeAgent> {} private final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); + private final Runnable aclMaintainer = mock(Runnable.class); private final ManualClock clock = new ManualClock(); - private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentFactory, Optional.empty(), + private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentFactory, aclMaintainer, new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock); @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java index 82f2408b252..fd54773910d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; @@ -15,6 +15,7 @@ import org.junit.Test; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -43,31 +44,33 @@ import static org.mockito.Mockito.when; public class NodeAdminStateUpdaterImplTest { private final NodeRepository nodeRepository = mock(NodeRepository.class); private final Orchestrator orchestrator = mock(Orchestrator.class); + private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); private final NodeAdmin nodeAdmin = mock(NodeAdmin.class); - private final HostName hostHostname = HostName.from("basehost1.test.yahoo.com"); + private final String parentHostname = "basehost1.test.yahoo.com"; private final ManualClock clock = new ManualClock(); private final Duration convergeStateInterval = Duration.ofSeconds(30); private final NodeAdminStateUpdaterImpl refresher = spy(new NodeAdminStateUpdaterImpl( - nodeRepository, orchestrator, nodeAdmin, hostHostname, clock, convergeStateInterval)); + nodeRepository, orchestrator, storageMaintainer, nodeAdmin, parentHostname, clock, + convergeStateInterval, Optional.empty())); @Test public void testStateConvergence() { mockNodeRepo(Node.State.active, 4); - List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() + List<String> activeHostnames = nodeRepository.getNodes(parentHostname).stream() .map(NodeSpec::getHostname) .collect(Collectors.toList()); List<String> suspendHostnames = new ArrayList<>(activeHostnames); - suspendHostnames.add(hostHostname.value()); + suspendHostnames.add(parentHostname); // Initially everything is frozen to force convergence assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, TRANSITION_EXCEPTION_MESSAGE); when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); - doNothing().when(orchestrator).resume(hostHostname.value()); + doNothing().when(orchestrator).resume(parentHostname); tickAfter(0); // The first tick should unfreeze - verify(orchestrator, times(1)).resume(hostHostname.value()); // Resume host - verify(orchestrator, times(1)).resume(hostHostname.value()); + verify(orchestrator, times(1)).resume(parentHostname); // Resume host + verify(orchestrator, times(1)).resume(parentHostname); // Everything is running and we want to continue running, therefore we have converged refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); @@ -75,7 +78,7 @@ public class NodeAdminStateUpdaterImplTest { tickAfter(35); refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); verify(refresher, never()).signalWorkToBeDone(); // No attempt in changing state - verify(orchestrator, times(1)).resume(hostHostname.value()); // Already resumed + verify(orchestrator, times(1)).resume(parentHostname); // Already resumed // Lets try to suspend node admin only, immediately we get false back, and need to wait until next // tick before any change can happen @@ -97,7 +100,7 @@ public class NodeAdminStateUpdaterImplTest { when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); doThrow(new RuntimeException(exceptionMessage)) - .when(orchestrator).suspend(eq(hostHostname.value())); + .when(orchestrator).suspend(eq(parentHostname)); tickAfter(35); assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, exceptionMessage); verify(refresher, times(1)).signalWorkToBeDone(); @@ -108,7 +111,7 @@ public class NodeAdminStateUpdaterImplTest { when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(NodeAdminStateUpdaterImpl.FREEZE_CONVERGENCE_TIMEOUT.plusMinutes(1)); doThrow(new RuntimeException(exceptionMessage)).doNothing() - .when(orchestrator).suspend(eq(hostHostname.value())); + .when(orchestrator).suspend(eq(parentHostname)); tickAfter(35); assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, exceptionMessage); verify(refresher, times(1)).signalWorkToBeDone(); @@ -119,12 +122,12 @@ public class NodeAdminStateUpdaterImplTest { verify(nodeAdmin, times(2)).setFrozen(eq(false)); // At this point orchestrator will say its OK to suspend, but something goes wrong when we try to stop services - verify(orchestrator, times(0)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); + verify(orchestrator, times(0)).suspend(eq(parentHostname), eq(suspendHostnames)); doThrow(new RuntimeException("Failed to stop services")).doNothing().when(nodeAdmin).stopNodeAgentServices(eq(activeHostnames)); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); // Change in wanted state, no need to wait - verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); + verify(orchestrator, times(1)).suspend(eq(parentHostname), eq(suspendHostnames)); verify(refresher, times(2)).signalWorkToBeDone(); // No change in desired state // Make sure we dont roll back if we fail to stop services - we will try to stop again next tick verify(nodeAdmin, times(2)).setFrozen(eq(false)); @@ -147,8 +150,8 @@ public class NodeAdminStateUpdaterImplTest { tickAfter(35); assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, "NodeAdmin is not yet unfrozen"); - doThrow(new OrchestratorException("Cannot allow to suspend " + hostHostname.value())).doNothing() - .when(orchestrator).resume(hostHostname.value()); + doThrow(new OrchestratorException("Cannot allow to suspend " + parentHostname)).doNothing() + .when(orchestrator).resume(parentHostname); tickAfter(35); assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, "Cannot allow to suspend basehost1.test.yahoo.com"); tickAfter(35); @@ -162,7 +165,7 @@ public class NodeAdminStateUpdaterImplTest { // Initially everything is frozen to force convergence when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); - doNothing().when(orchestrator).resume(hostHostname.value()); + doNothing().when(orchestrator).resume(parentHostname); tickAfter(0); // The first tick should unfreeze refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); verify(nodeAdmin, times(1)).setFrozen(eq(false)); @@ -170,7 +173,7 @@ public class NodeAdminStateUpdaterImplTest { // Let's start suspending, we are able to freeze the nodes, but orchestrator denies suspension when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); - doThrow(new RuntimeException(exceptionMsg)).when(orchestrator).suspend(eq(hostHostname.value())); + doThrow(new RuntimeException(exceptionMsg)).when(orchestrator).suspend(eq(parentHostname)); assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, TRANSITION_EXCEPTION_MESSAGE); tickAfter(30); @@ -204,21 +207,21 @@ public class NodeAdminStateUpdaterImplTest { // orchestrator to resume/suspend host. Therefore, if host is not active, we only need to freeze. tickAfter(0); refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); - verify(orchestrator, never()).resume(eq(hostHostname.value())); + verify(orchestrator, never()).resume(eq(parentHostname)); assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN); - verify(orchestrator, never()).suspend(eq(hostHostname.value())); + verify(orchestrator, never()).suspend(eq(parentHostname)); // When doing batch suspend, only suspend the containers if the host is not active - List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() + List<String> activeHostnames = nodeRepository.getNodes(parentHostname).stream() .map(NodeSpec::getHostname) .collect(Collectors.toList()); assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); - verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(activeHostnames)); + verify(orchestrator, times(1)).suspend(eq(parentHostname), eq(activeHostnames)); } private void assertResumeStateError(NodeAdminStateUpdater.State targetState, String reason) { @@ -243,10 +246,10 @@ public class NodeAdminStateUpdaterImplTest { .build()) .collect(Collectors.toList()); - when(nodeRepository.getNodes(eq(hostHostname.value()))).thenReturn(containersToRun); + when(nodeRepository.getNodes(eq(parentHostname))).thenReturn(containersToRun); - when(nodeRepository.getNode(eq(hostHostname.value()))).thenReturn(new NodeSpec.Builder() - .hostname(hostHostname.value()) + when(nodeRepository.getNode(eq(parentHostname))).thenReturn(new NodeSpec.Builder() + .hostname(parentHostname) .state(hostState) .nodeType(NodeType.tenant) .flavor("default") diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java index 84f13ed299a..31ac8d1c114 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java @@ -1,9 +1,13 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.Paths; import static org.junit.Assert.assertEquals; @@ -12,8 +16,7 @@ import static org.junit.Assert.assertEquals; */ public class NodeAgentContextImplTest { private final FileSystem fileSystem = TestFileSystem.create(); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-1.domain.tld") - .fileSystem(fileSystem).build(); + private final NodeAgentContext context = nodeAgentFromHostname(fileSystem, "container-1.domain.tld"); @Test public void path_on_host_from_path_in_node_test() { @@ -68,4 +71,18 @@ public class NodeAgentContextImplTest { public void path_under_vespa_home_must_be_relative() { context.pathInNodeUnderVespaHome("/home"); } + + + public static NodeAgentContext nodeAgentFromHostname(String hostname) { + FileSystem fileSystem = TestFileSystem.create(); + return nodeAgentFromHostname(fileSystem, hostname); + } + + public static NodeAgentContext nodeAgentFromHostname(FileSystem fileSystem, String hostname) { + final Path vespaHomeInContainer = Paths.get("/opt/vespa"); + final Path containerStoragePath = fileSystem.getPath("/home/docker"); + + return new NodeAgentContextImpl(hostname, NodeType.tenant, new AthenzService("domain", "service"), + containerStoragePath, vespaHomeInContainer); + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 00326f90caa..5cc881fe62d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -12,17 +12,22 @@ import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; import org.mockito.InOrder; +import java.io.IOException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -63,7 +68,7 @@ public class NodeAgentImplTest { private static final String vespaVersion = "1.2.3"; private final String hostName = "host1.test.yahoo.com"; - private final NodeAgentContext context = new NodeAgentContextImpl.Builder(hostName).build(); + private final NodeAgentContext context = NodeAgentContextImplTest.nodeAgentFromHostname(hostName); private final DockerImage dockerImage = new DockerImage("dockerImage"); private final DockerOperations dockerOperations = mock(DockerOperations.class); private final NodeRepository nodeRepository = mock(NodeRepository.class); @@ -75,7 +80,18 @@ public class NodeAgentImplTest { Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); private final AthenzCredentialsMaintainer athenzCredentialsMaintainer = mock(AthenzCredentialsMaintainer.class); + private final PathResolver pathResolver = mock(PathResolver.class); private final ManualClock clock = new ManualClock(); + private final Environment environment = new Environment.Builder() + .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) + .environment("dev") + .region("us-east-1") + .system("main") + .cloud("mycloud") + .parentHostHostname("parent.host.name.yahoo.com") + .pathResolver(pathResolver) + .dockerNetworking(DockerNetworking.HOST_NETWORK) + .build(); private final NodeSpec.Builder nodeBuilder = new NodeSpec.Builder() .hostname(context.hostname().value()) @@ -180,7 +196,7 @@ public class NodeAgentImplTest { } @Test - public void absentContainerCausesStart() { + public void absentContainerCausesStart() throws Exception { final Optional<Long> restartGeneration = Optional.of(1L); final long rebootGeneration = 0; final NodeSpec node = nodeBuilder @@ -195,6 +211,8 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo")); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); @@ -208,7 +226,7 @@ public class NodeAgentImplTest { inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage)); inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); inOrder.verify(dockerOperations, times(1)).startContainer(eq(context)); - inOrder.verify(aclMaintainer, times(1)).converge(); + inOrder.verify(aclMaintainer, times(1)).run(); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( hostName, new NodeAttributes() @@ -250,7 +268,7 @@ public class NodeAgentImplTest { } @Test - public void containerIsRestartedIfFlavorChanged() { + public void containerIsRestartedIfFlavorChanged() throws IOException { final long wantedRestartGeneration = 1; final long currentRestartGeneration = 1; NodeSpec.Builder specBuilder = nodeBuilder @@ -273,6 +291,7 @@ public class NodeAgentImplTest { .thenReturn(Optional.of(thirdSpec)); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); nodeAgent.converge(); nodeAgent.converge(); @@ -482,7 +501,7 @@ public class NodeAgentImplTest { } @Test - public void testRestartDeadContainerAfterNodeAdminRestart() { + public void testRestartDeadContainerAfterNodeAdminRestart() throws IOException { final NodeSpec node = nodeBuilder .currentDockerImage(dockerImage) .wantedDockerImage(dockerImage) @@ -493,6 +512,8 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); + when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo")); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.tick(); @@ -569,7 +590,7 @@ public class NodeAgentImplTest { } @Test - public void start_container_subtask_failure_leads_to_container_restart() { + public void start_container_subtask_failure_leads_to_container_restart() throws IOException { final long restartGeneration = 1; final long rebootGeneration = 0; final NodeSpec node = nodeBuilder @@ -584,6 +605,8 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false)); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo")); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context)); @@ -636,7 +659,6 @@ public class NodeAgentImplTest { .membership(membership) .minMainMemoryAvailableGb(2) .allowedToBeDown(true) - .parentHostname("parent.host.name.yahoo.com") .build(); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -646,6 +668,7 @@ public class NodeAgentImplTest { when(dockerOperations.getContainerStats(eq(context))) .thenReturn(Optional.of(stats1)) .thenReturn(Optional.of(stats2)); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); nodeAgent.converge(); // Run the converge loop once to initialize lastNode nodeAgent.updateContainerNodeMetrics(); // Update metrics once to init and lastCpuMetric @@ -694,7 +717,7 @@ public class NodeAgentImplTest { } @Test - public void testRunningConfigServer() { + public void testRunningConfigServer() throws IOException { final long rebootGeneration = 0; final NodeSpec node = nodeBuilder .nodeType(NodeType.config) @@ -706,6 +729,8 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + Path tempDirectory = Files.createTempDirectory("foo"); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(tempDirectory); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); @@ -718,7 +743,7 @@ public class NodeAgentImplTest { inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage)); inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); inOrder.verify(dockerOperations, times(1)).startContainer(eq(context)); - inOrder.verify(aclMaintainer, times(1)).converge(); + inOrder.verify(aclMaintainer, times(1)).run(); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( hostName, new NodeAttributes() @@ -731,10 +756,11 @@ public class NodeAgentImplTest { mockGetContainer(dockerImage, isRunning); when(dockerOperations.getContainerStats(any())).thenReturn(Optional.of(emptyContainerStats)); + doNothing().when(storageMaintainer).writeFilebeatConfig(any(), any()); doNothing().when(storageMaintainer).writeMetricsConfig(any(), any()); return new NodeAgentImpl(context, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.of(athenzCredentialsMaintainer), Optional.of(aclMaintainer)); + storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL, Optional.of(athenzCredentialsMaintainer)); } private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { |