diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2018-10-18 14:13:34 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-18 14:13:34 +0200 |
commit | 20f45acb393b7973fcae56b9bbcd05faf340e56c (patch) | |
tree | 1578380b3abf06a976d5338367423f4651bf59f8 /node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin | |
parent | e20cb83a53bcd756afb067db9a9ab030ed6d679d (diff) |
Revert "NodeAdmin: Remove usage for Environment"
Diffstat (limited to 'node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin')
12 files changed, 247 insertions, 216 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java index f1218efa67c..1b4ddf6aedd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java @@ -16,10 +16,9 @@ import java.util.Map; * * @author hmusum */ -@FunctionalInterface public interface ContainerEnvironmentResolver { - String createSettings(NodeSpec node); + String createSettings(Environment environment, NodeSpec node); class ContainerEnvironmentSettings { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java new file mode 100644 index 00000000000..38f57c9544a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java @@ -0,0 +1,18 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.component; + +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; + +/** + * @author hmusum + */ +public class DefaultContainerEnvironmentResolver implements ContainerEnvironmentResolver { + + public String createSettings(Environment environment, NodeSpec node) { + return new ContainerEnvironmentSettings() + .set("nodeType", node.getNodeType()) + .set("cloud", environment.getCloud()) + .build(); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java index aaadd3cb24e..857f796fb5c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java @@ -307,7 +307,7 @@ public class Environment { Optional.ofNullable(pathResolver).orElseGet(PathResolver::new), logstashNodes, nodeType, - Optional.ofNullable(containerEnvironmentResolver).orElseGet(() -> node -> ""), + Optional.ofNullable(containerEnvironmentResolver).orElseGet(DefaultContainerEnvironmentResolver::new), certificateDnsSuffix, ztsUri, nodeAthenzIdentity, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ZoneId.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ZoneId.java deleted file mode 100644 index 02f6da2e0e3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ZoneId.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; - -import java.util.Objects; - -/** - * @author freva - */ -public class ZoneId { - private final SystemName systemName; - private final Environment environment; - private final RegionName regionName; - - public ZoneId(SystemName systemName, Environment environment, RegionName regionName) { - this.systemName = systemName; - this.environment = environment; - this.regionName = regionName; - } - - public SystemName systemName() { - return systemName; - } - - public Environment environment() { - return environment; - } - - public RegionName regionName() { - return regionName; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ZoneId zoneId = (ZoneId) o; - return systemName == zoneId.systemName && - environment == zoneId.environment && - Objects.equals(regionName, zoneId.regionName); - } - - @Override - public int hashCode() { - return Objects.hash(systemName, environment, regionName); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 7a83f00e297..f1fc6776380 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.node.admin.component.ContainerEnvironmentResolver; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; @@ -46,19 +46,13 @@ public class DockerOperationsImpl implements DockerOperations { private static final String IPV4_NPT_PREFIX = "172.17.0.0"; private final Docker docker; + private final Environment environment; private final ProcessExecuter processExecuter; - private final ContainerEnvironmentResolver containerEnvironmentResolver; - private final List<String> configServerHostnames; - private final IPAddresses ipAddresses; - public DockerOperationsImpl(Docker docker, ProcessExecuter processExecuter, - ContainerEnvironmentResolver containerEnvironmentResolver, - List<String> configServerHostnames, IPAddresses ipAddresses) { + public DockerOperationsImpl(Docker docker, Environment environment, ProcessExecuter processExecuter) { this.docker = docker; + this.environment = environment; this.processExecuter = processExecuter; - this.containerEnvironmentResolver = containerEnvironmentResolver; - this.configServerHostnames = configServerHostnames; - this.ipAddresses = ipAddresses; } @Override @@ -66,11 +60,11 @@ public class DockerOperationsImpl implements DockerOperations { context.log(logger, "Creating container"); // IPv6 - Assume always valid - Inet6Address ipV6Address = ipAddresses.getIPv6Address(node.getHostname()).orElseThrow( + Inet6Address ipV6Address = environment.getIpAddresses().getIPv6Address(node.getHostname()).orElseThrow( () -> new RuntimeException("Unable to find a valid IPv6 address for " + node.getHostname() + ". Missing an AAAA DNS entry?")); - String configServers = String.join(",", configServerHostnames); + String configServers = String.join(",", environment.getConfigServerHostNames()); Docker.CreateContainerCommand command = docker.createContainerCommand( node.getWantedDockerImage().get(), @@ -79,7 +73,8 @@ public class DockerOperationsImpl implements DockerOperations { node.getHostname()) .withManagedBy(MANAGER_NAME) .withEnvironment("VESPA_CONFIGSERVERS", configServers) - .withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS", containerEnvironmentResolver.createSettings(node)) + .withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS", + environment.getContainerEnvironmentResolver().createSettings(environment, node)) .withUlimit("nofile", 262_144, 262_144) .withUlimit("nproc", 32_768, 409_600) .withUlimit("core", -1, -1) @@ -87,7 +82,7 @@ public class DockerOperationsImpl implements DockerOperations { .withAddCapability("SYS_ADMIN"); // Needed for perf - DockerNetworking networking = context.dockerNetworking(); + DockerNetworking networking = environment.getDockerNetworking(); command.withNetworkMode(networking.getDockerNetworkMode()); if (networking == DockerNetworking.NPT) { @@ -96,7 +91,7 @@ public class DockerOperationsImpl implements DockerOperations { command.withIpAddress(ipV6Local); // IPv4 - Only present for some containers - Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(node.getHostname()) + Optional<InetAddress> ipV4Local = environment.getIpAddresses().getIPv4Address(node.getHostname()) .map(ipV4Address -> { InetAddress ipV4Prefix = InetAddresses.forString(IPV4_NPT_PREFIX); return IPAddresses.prefixTranslate(ipV4Address, ipV4Prefix, 2); @@ -270,6 +265,7 @@ public class DockerOperationsImpl implements DockerOperations { // Paths unique to each container List<Path> paths = new ArrayList<>(Arrays.asList( Paths.get("/etc/yamas-agent"), + Paths.get("/etc/filebeat"), context.pathInNodeUnderVespaHome("logs/daemontools_y"), context.pathInNodeUnderVespaHome("logs/jdisc_core"), context.pathInNodeUnderVespaHome("logs/langdetect/"), diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 2b235ac9867..171e47d33c5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -1,12 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance; +import com.yahoo.collections.Pair; import com.yahoo.config.provision.NodeType; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; +import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -23,6 +28,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -31,7 +37,9 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.regex.Pattern; +import java.util.stream.Stream; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.olderThan; import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists; @@ -46,11 +54,16 @@ public class StorageMaintainer { .ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC); private final DockerOperations dockerOperations; + private final ProcessExecuter processExecuter; + private final Environment environment; private final CoredumpHandler coredumpHandler; private final Path archiveContainerStoragePath; - public StorageMaintainer(DockerOperations dockerOperations, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) { + public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter, + Environment environment, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) { this.dockerOperations = dockerOperations; + this.processExecuter = processExecuter; + this.environment = environment; this.coredumpHandler = coredumpHandler; this.archiveContainerStoragePath = archiveContainerStoragePath; } @@ -61,37 +74,37 @@ public class StorageMaintainer { // host-life Path hostLifeCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life"); SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath); - configs.add(annotatedCheck(context, node, hostLifeSchedule)); + configs.add(annotatedCheck(node, hostLifeSchedule)); // ntp Path ntpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ntp"); SecretAgentCheckConfig ntpSchedule = new SecretAgentCheckConfig("ntp", 60, ntpCheckPath); - configs.add(annotatedCheck(context, node, ntpSchedule)); + configs.add(annotatedCheck(node, ntpSchedule)); // coredumps (except for the done coredumps which is handled by the host) Path coredumpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps"); SecretAgentCheckConfig coredumpSchedule = new SecretAgentCheckConfig("system-coredumps-processing", 300, coredumpCheckPath, "--application", "system-coredumps-processing", "--lastmin", "129600", "--crit", "1", "--coredir", context.pathInNodeUnderVespaHome("var/crash/processing").toString()); - configs.add(annotatedCheck(context, node, coredumpSchedule)); + configs.add(annotatedCheck(node, coredumpSchedule)); // athenz certificate check Path athenzCertExpiryCheckPath = context.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs"); SecretAgentCheckConfig athenzCertExpirySchedule = new SecretAgentCheckConfig("athenz-certificate-expiry", 60, athenzCertExpiryCheckPath, "--threshold", "20") .withRunAsUser("root"); - configs.add(annotatedCheck(context, node, athenzCertExpirySchedule)); + configs.add(annotatedCheck(node, athenzCertExpirySchedule)); if (context.nodeType() != NodeType.config) { // vespa-health Path vespaHealthCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health"); SecretAgentCheckConfig vespaHealthSchedule = new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all"); - configs.add(annotatedCheck(context, node, vespaHealthSchedule)); + configs.add(annotatedCheck(node, vespaHealthSchedule)); // vespa Path vespaCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa"); SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all"); - configs.add(annotatedCheck(context, node, vespaSchedule)); + configs.add(annotatedCheck(node, vespaSchedule)); } if (context.nodeType() == NodeType.config) { @@ -99,14 +112,14 @@ public class StorageMaintainer { Path configServerCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ymonsb2"); SecretAgentCheckConfig configServerSchedule = new SecretAgentCheckConfig("configserver", 60, configServerCheckPath, "-zero", "configserver"); - configs.add(annotatedCheck(context, node, configServerSchedule)); + configs.add(annotatedCheck(node, configServerSchedule)); //zkbackupage Path zkbackupCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); SecretAgentCheckConfig zkbackupSchedule = new SecretAgentCheckConfig("zkbackupage", 300, zkbackupCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(), "-m", "150", "-a", "config-zkbackupage"); - configs.add(annotatedCheck(context, node, zkbackupSchedule)); + configs.add(annotatedCheck(node, zkbackupSchedule)); } if (context.nodeType() == NodeType.proxy) { @@ -115,13 +128,13 @@ public class StorageMaintainer { SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60, routingAgeCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(), "-m", "90", "-a", "routing-configage"); - configs.add(annotatedCheck(context, node, routingAgeSchedule)); + configs.add(annotatedCheck(node, routingAgeSchedule)); //ssl-check Path sslCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status"); SecretAgentCheckConfig sslSchedule = new SecretAgentCheckConfig("ssl-status", 300, sslCheckPath, "-e", "localhost", "-p", "4443", "-t", "30"); - configs.add(annotatedCheck(context, node, sslSchedule)); + configs.add(annotatedCheck(node, sslSchedule)); } // Write config and restart yamas-agent @@ -134,14 +147,14 @@ public class StorageMaintainer { dockerOperations.executeCommandInContainerAsRoot(context, "service", "yamas-agent", "restart"); } - private SecretAgentCheckConfig annotatedCheck(NodeAgentContext context, NodeSpec node, SecretAgentCheckConfig check) { + private SecretAgentCheckConfig annotatedCheck(NodeSpec node, SecretAgentCheckConfig check) { check.withTag("namespace", "Vespa") .withTag("role", SecretAgentCheckConfig.nodeTypeToRole(node.getNodeType())) .withTag("flavor", node.getFlavor()) .withTag("canonicalFlavor", node.getCanonicalFlavor()) .withTag("state", node.getState().toString()) - .withTag("zone", String.format("%s.%s", context.zoneId().environment().value(), context.zoneId().regionName().value())); - node.getParentHostname().ifPresent(parent -> check.withTag("parentHostname", parent)); + .withTag("zone", environment.getZone()) + .withTag("parentHostname", environment.getParentHostHostname()); node.getOwner().ifPresent(owner -> check .withTag("tenantName", owner.getTenant()) .withTag("app", owner.getApplication() + "." + owner.getInstance()) @@ -156,6 +169,20 @@ public class StorageMaintainer { return check; } + public void writeFilebeatConfig(NodeAgentContext context, NodeSpec node) { + try { + FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment); + Optional<String> config = filebeatConfigProvider.getConfig(context, node); + if (!config.isPresent()) return; + + Path filebeatPath = context.pathOnHostFromPathInNode("/etc/filebeat/filebeat.yml"); + Files.write(filebeatPath, config.get().getBytes()); + context.log(logger, "Wrote filebeat config"); + } catch (Throwable t) { + context.log(logger, LogLevel.ERROR, "Failed writing filebeat config", t); + } + } + public Optional<Long> getDiskUsageFor(NodeAgentContext context) { Path containerDir = context.pathOnHostFromPathInNode("/"); try { @@ -220,20 +247,20 @@ public class StorageMaintainer { /** Checks if container has any new coredumps, reports and archives them if so */ public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node, Optional<Container> container) { - final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(context, node, container); + final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(node, container); coredumpHandler.converge(context, nodeAttributes); } - private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, NodeSpec node, Optional<Container> container) { + private Map<String, Object> getCoredumpNodeAttributes(NodeSpec node, Optional<Container> container) { Map<String, Object> attributes = new HashMap<>(); attributes.put("hostname", node.getHostname()); - attributes.put("region", context.zoneId().regionName()); - attributes.put("environment", context.zoneId().environment()); + attributes.put("parent_hostname", environment.getParentHostHostname()); + attributes.put("region", environment.getRegion()); + attributes.put("environment", environment.getEnvironment()); attributes.put("flavor", node.getFlavor()); attributes.put("kernel_version", System.getProperty("os.version")); container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString())); - node.getParentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); node.getVespaVersion().ifPresent(version -> attributes.put("vespa_version", version)); node.getOwner().ifPresent(owner -> { attributes.put("tenant", owner.getTenant()); @@ -257,4 +284,56 @@ public class StorageMaintainer { new UnixPath(containerLogsOnHost).moveIfExists(containerLogsInArchiveDir); new UnixPath(context.pathOnHostFromPathInNode("/")).deleteRecursively(); } + + /** + * Runs node-maintainer's SpecVerifier and returns its output + * @param node Node specification containing the excepted values we want to verify against + * @return new combined hardware divergence + * @throws RuntimeException if exit code != 0 + */ + public String getHardwareDivergence(NodeSpec node) { + List<String> arguments = new ArrayList<>(Arrays.asList("specification", + "--disk", Double.toString(node.getMinDiskAvailableGb()), + "--memory", Double.toString(node.getMinMainMemoryAvailableGb()), + "--cpu_cores", Double.toString(node.getMinCpuCores()), + "--is_ssd", Boolean.toString(node.isFastDisk()), + "--bandwidth", Double.toString(node.getBandwidth()), + "--ips", String.join(",", node.getIpAddresses()))); + + if (environment.getDockerNetworking() == DockerNetworking.HOST_NETWORK) { + arguments.add("--skip-reverse-lookup"); + } + + node.getHardwareDivergence().ifPresent(hardwareDivergence -> { + arguments.add("--divergence"); + arguments.add(hardwareDivergence); + }); + + return executeMaintainer("com.yahoo.vespa.hosted.node.verification.Main", arguments.toArray(new String[0])); + } + + + private String executeMaintainer(String mainClass, String... args) { + String[] command = Stream.concat( + Stream.of("sudo", + "VESPA_HOSTNAME=" + getDefaults().vespaHostname(), + "VESPA_HOME=" + getDefaults().vespaHome(), + getDefaults().underVespaHome("libexec/vespa/node-admin/maintenance.sh"), + mainClass), + Stream.of(args)) + .toArray(String[]::new); + + try { + Pair<Integer, String> result = processExecuter.exec(command); + + if (result.getFirst() != 0) { + throw new RuntimeException( + String.format("Maintainer failed to execute command: %s, Exit code: %d, Stdout/stderr: %s", + Arrays.toString(command), result.getFirst(), result.getSecond())); + } + return result.getSecond().trim(); + } catch (IOException e) { + throw new RuntimeException("Failed to execute maintainer", e); + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index a733a52ea10..88fb5dcb90f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -2,10 +2,11 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.acl; import com.google.common.net.InetAddresses; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.dockerapi.Container; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; @@ -30,21 +31,23 @@ import java.util.stream.Collectors; * @author mpolden * @author smorgrav */ -public class AclMaintainer { +public class AclMaintainer implements Runnable { private static final PrefixLogger log = PrefixLogger.getNodeAdminLogger(AclMaintainer.class); private final DockerOperations dockerOperations; private final NodeRepository nodeRepository; private final IPAddresses ipAddresses; - private final String hostHostname; + private final String nodeAdminHostname; + private final Environment environment; public AclMaintainer(DockerOperations dockerOperations, NodeRepository nodeRepository, - HostName hostHostname, IPAddresses ipAddresses) { + String nodeAdminHostname, IPAddresses ipAddresses, Environment environment) { this.dockerOperations = dockerOperations; this.nodeRepository = nodeRepository; this.ipAddresses = ipAddresses; - this.hostHostname = hostHostname.value(); + this.nodeAdminHostname = nodeAdminHostname; + this.environment = environment; } private void applyRedirect(Container container, InetAddress address) { @@ -65,18 +68,21 @@ public class AclMaintainer { } private synchronized void configureAcls() { + if (environment.getDockerNetworking() != DockerNetworking.NPT) return; + log.info("Configuring ACLs"); // Needed to potentially nail down when ACL maintainer stopped working Map<String, Container> runningContainers = dockerOperations .getAllManagedContainers().stream() .filter(container -> container.state.isRunning()) .collect(Collectors.toMap(container -> container.hostname, container -> container)); - nodeRepository.getAcls(hostHostname).entrySet().stream() + nodeRepository.getAcls(nodeAdminHostname).entrySet().stream() .filter(entry -> runningContainers.containsKey(entry.getKey())) .forEach(entry -> apply(runningContainers.get(entry.getKey()), entry.getValue())); } - public void converge() { + @Override + public void run() { try { configureAcls(); } catch (Throwable t) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index b1c1a99a90a..d436e214266 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; @@ -18,7 +18,6 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -40,7 +39,7 @@ public class NodeAdminImpl implements NodeAdmin { Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler")); private final Function<String, NodeAgent> nodeAgentFactory; - private final Optional<AclMaintainer> aclMaintainer; + private final Runnable aclMaintainer; private final Clock clock; private boolean previousWantFrozen; @@ -52,8 +51,16 @@ public class NodeAdminImpl implements NodeAdmin { private final GaugeWrapper numberOfContainersInLoadImageState; private final CounterWrapper numberOfUnhandledExceptionsInNodeAgent; + public NodeAdminImpl(DockerOperations dockerOperations, + Function<String, NodeAgent> nodeAgentFactory, + Runnable aclMaintainer, + MetricReceiverWrapper metricReceiver, + Clock clock) { + this(nodeAgentFactory, aclMaintainer, metricReceiver, clock); + } + public NodeAdminImpl(Function<String, NodeAgent> nodeAgentFactory, - Optional<AclMaintainer> aclMaintainer, + Runnable aclMaintainer, MetricReceiverWrapper metricReceiver, Clock clock) { this.nodeAgentFactory = nodeAgentFactory; @@ -168,12 +175,10 @@ public class NodeAdminImpl implements NodeAdmin { } }, 10, 55, TimeUnit.SECONDS); - aclMaintainer.ifPresent(maintainer -> { - int delay = 120; // WARNING: Reducing this will increase the load on config servers. - aclScheduler.scheduleWithFixedDelay(() -> { - if (!isFrozen()) maintainer.converge(); - }, 30, delay, TimeUnit.SECONDS); - }); + int delay = 120; // WARNING: Reducing this will increase the load on config servers. + aclScheduler.scheduleWithFixedDelay(() -> { + if (!isFrozen()) aclMaintainer.run(); + }, 30, delay, TimeUnit.SECONDS); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java index 296745c8e37..5b0a3c7771f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; -import com.yahoo.config.provision.HostName; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.concurrent.classlock.ClassLock; +import com.yahoo.concurrent.classlock.ClassLocking; +import com.yahoo.concurrent.classlock.LockInterruptException; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; @@ -19,7 +24,11 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -45,34 +54,55 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { private RuntimeException lastConvergenceException; private final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); + private final ScheduledExecutorService specVerifierScheduler = + Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("specverifier")); private final Thread loopThread; private final NodeRepository nodeRepository; private final Orchestrator orchestrator; private final NodeAdmin nodeAdmin; private final Clock clock; - private final String hostHostname; + private final String dockerHostHostName; private final Duration nodeAdminConvergeStateInterval; + private final Optional<ClassLocking> classLocking; + private Optional<ClassLock> classLock = Optional.empty(); private Instant lastTick; public NodeAdminStateUpdaterImpl( NodeRepository nodeRepository, Orchestrator orchestrator, + StorageMaintainer storageMaintainer, NodeAdmin nodeAdmin, - HostName hostHostname, + String dockerHostHostName, Clock clock, - Duration nodeAdminConvergeStateInterval) { + Duration nodeAdminConvergeStateInterval, + Optional<ClassLocking> classLocking) { + log.info(objectToString() + ": Creating object"); this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; this.nodeAdmin = nodeAdmin; - this.hostHostname = hostHostname.value(); + this.dockerHostHostName = dockerHostHostName; this.clock = clock; this.nodeAdminConvergeStateInterval = nodeAdminConvergeStateInterval; + this.classLocking = classLocking; this.lastTick = clock.instant(); this.loopThread = new Thread(() -> { + if (classLocking.isPresent()) { + log.info(objectToString() + ": Acquiring lock"); + try { + classLock = Optional.of(classLocking.get().lockWhile(NodeAdminStateUpdater.class, () -> !terminated.get())); + } catch (LockInterruptException e) { + classLock = Optional.empty(); + return; + } + } + + log.info(objectToString() + ": Starting threads and schedulers"); nodeAdmin.start(); + specVerifierScheduler.scheduleWithFixedDelay(() -> + updateHardwareDivergence(storageMaintainer), 5, 60, TimeUnit.MINUTES); while (! terminated.get()) { tick(); @@ -81,11 +111,15 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { this.loopThread.setName("tick-NodeAdminStateUpdater"); } + private String objectToString() { + return this.getClass().getSimpleName() + "@" + Integer.toString(System.identityHashCode(this)); + } + @Override public Map<String, Object> getDebugPage() { Map<String, Object> debug = new LinkedHashMap<>(); synchronized (monitor) { - debug.put("hostHostname", hostHostname); + debug.put("dockerHostHostName", dockerHostHostName); debug.put("wantedState", wantedState); debug.put("currentState", currentState); debug.put("NodeAdmin", nodeAdmin.debugInfo()); @@ -93,6 +127,25 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { return debug; } + private void updateHardwareDivergence(StorageMaintainer maintainer) { + if (currentState != RESUMED) return; + + try { + NodeSpec node = nodeRepository.getNode(dockerHostHostName); + if (node.getState() == Node.State.parked) return; + String hardwareDivergence = maintainer.getHardwareDivergence(node); + + // Only update hardware divergence if there is a change. + if (!node.getHardwareDivergence().orElse("null").equals(hardwareDivergence)) { + NodeAttributes nodeAttributes = new NodeAttributes().withHardwareDivergence(hardwareDivergence); + log.info("Updating hardware divergence to " + hardwareDivergence); + nodeRepository.updateNodeAttributes(dockerHostHostName, nodeAttributes); + } + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to report hardware divergence", e); + } + } + @Override public void setResumeStateAndCheckIfResumed(State wantedState) { synchronized (monitor) { @@ -186,13 +239,13 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { throw new ConvergenceException("NodeAdmin is not yet " + (wantFrozen ? "frozen" : "unfrozen")); } - boolean hostIsActiveInNR = nodeRepository.getNode(hostHostname).getState() == Node.State.active; + boolean hostIsActiveInNR = nodeRepository.getNode(dockerHostHostName).getState() == Node.State.active; switch (wantedState) { case RESUMED: - if (hostIsActiveInNR) orchestrator.resume(hostHostname); + if (hostIsActiveInNR) orchestrator.resume(dockerHostHostName); break; case SUSPENDED_NODE_ADMIN: - if (hostIsActiveInNR) orchestrator.suspend(hostHostname); + if (hostIsActiveInNR) orchestrator.suspend(dockerHostHostName); break; case SUSPENDED: // Fetch active nodes from node repo before suspending nodes. @@ -204,9 +257,9 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { List<String> nodesInActiveState = getNodesInActiveState(); List<String> nodesToSuspend = new ArrayList<>(nodesInActiveState); - if (hostIsActiveInNR) nodesToSuspend.add(hostHostname); + if (hostIsActiveInNR) nodesToSuspend.add(dockerHostHostName); if (!nodesToSuspend.isEmpty()) { - orchestrator.suspend(hostHostname, nodesToSuspend); + orchestrator.suspend(dockerHostHostName, nodesToSuspend); log.info("Orchestrator allows suspension of " + nodesToSuspend); } @@ -228,7 +281,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { private void fetchContainersToRunFromNodeRepository() { try { - final List<NodeSpec> containersToRun = nodeRepository.getNodes(hostHostname); + final List<NodeSpec> containersToRun = nodeRepository.getNodes(dockerHostHostName); nodeAdmin.refreshContainersToRun(containersToRun); } catch (Exception e) { log.log(LogLevel.WARNING, "Failed to update which containers should be running", e); @@ -236,7 +289,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { } private List<String> getNodesInActiveState() { - return nodeRepository.getNodes(hostHostname) + return nodeRepository.getNodes(dockerHostHostName) .stream() .filter(node -> node.getState() == Node.State.active) .map(NodeSpec::getHostname) @@ -248,22 +301,33 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { } public void stop() { + log.info(objectToString() + ": Stop called"); if (!terminated.compareAndSet(false, true)) { throw new RuntimeException("Can not re-stop a node agent."); } + classLocking.ifPresent(ClassLocking::interrupt); + // First we need to stop NodeAdminStateUpdaterImpl thread to make sure no new NodeAgents are spawned signalWorkToBeDone(); + specVerifierScheduler.shutdown(); do { try { loopThread.join(); + specVerifierScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e1) { log.info("Interrupted while waiting for NodeAdminStateUpdater thread and specVerfierScheduler to shutdown"); } - } while (loopThread.isAlive()); + } while (loopThread.isAlive() || !specVerifierScheduler.isTerminated()); // Finally, stop NodeAdmin and all the NodeAgents nodeAdmin.stop(); + + classLock.ifPresent(lock -> { + log.info(objectToString() + ": Releasing lock"); + lock.close(); + }); + log.info(objectToString() + ": Stop complete"); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index 2025d422331..f65f371ff67 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.component.ZoneId; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import java.nio.file.Path; import java.nio.file.Paths; @@ -21,10 +19,6 @@ public interface NodeAgentContext extends TaskContext { AthenzService identity(); - DockerNetworking dockerNetworking(); - - ZoneId zoneId(); - /** * This method is the inverse of {@link #pathInNodeFromPathOnHost(Path)}} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index d1b5ec77c6a..d3c8b145488 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -1,20 +1,13 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.node.admin.component.ZoneId; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import java.nio.file.FileSystem; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Objects; -import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -29,20 +22,15 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final HostName hostName; private final NodeType nodeType; private final AthenzService identity; - private final DockerNetworking dockerNetworking; - private final ZoneId zoneId; private final Path pathToNodeRootOnHost; private final Path pathToVespaHome; public NodeAgentContextImpl(String hostname, NodeType nodeType, AthenzService identity, - DockerNetworking dockerNetworking, ZoneId zoneId, Path pathToContainerStorage, Path pathToVespaHome) { this.hostName = HostName.from(Objects.requireNonNull(hostname)); this.containerName = ContainerName.fromHostname(hostname); this.nodeType = Objects.requireNonNull(nodeType); this.identity = Objects.requireNonNull(identity); - this.dockerNetworking = Objects.requireNonNull(dockerNetworking); - this.zoneId = Objects.requireNonNull(zoneId); this.pathToNodeRootOnHost = Objects.requireNonNull(pathToContainerStorage).resolve(containerName.asString()); this.pathToVespaHome = Objects.requireNonNull(pathToVespaHome); this.logPrefix = containerName.asString() + ": "; @@ -69,16 +57,6 @@ public class NodeAgentContextImpl implements NodeAgentContext { } @Override - public DockerNetworking dockerNetworking() { - return dockerNetworking; - } - - @Override - public ZoneId zoneId() { - return zoneId; - } - - @Override public Path pathOnHostFromPathInNode(Path pathInNode) { if (! pathInNode.isAbsolute()) throw new IllegalArgumentException("Expected an absolute path in the container, got: " + pathInNode); @@ -119,66 +97,4 @@ public class NodeAgentContextImpl implements NodeAgentContext { public void log(Logger logger, Level level, String message, Throwable throwable) { logger.log(level, logPrefix + message, throwable); } - - - /** For testing only! */ - public static class Builder { - private final String hostname; - private NodeType nodeType; - private AthenzService identity; - private DockerNetworking dockerNetworking; - private ZoneId zoneId; - private Path pathToContainerStorage; - private Path pathToVespaHome; - - public Builder(String hostname) { - this.hostname = hostname; - } - - public Builder nodeType(NodeType nodeType) { - this.nodeType = nodeType; - return this; - } - - public Builder identity(AthenzService identity) { - this.identity = identity; - return this; - } - - public Builder dockerNetworking(DockerNetworking dockerNetworking) { - this.dockerNetworking = dockerNetworking; - return this; - } - - public Builder zoneId(ZoneId zoneId) { - this.zoneId = zoneId; - return this; - } - - public Builder pathToContainerStorage(Path pathToContainerStorage) { - this.pathToContainerStorage = pathToContainerStorage; - return this; - } - - public Builder pathToVespaHome(Path pathToVespaHome) { - this.pathToVespaHome = pathToVespaHome; - return this; - } - - public Builder fileSystem(FileSystem fileSystem) { - return pathToContainerStorage(fileSystem.getPath("/home/docker")); - } - - public NodeAgentContextImpl build() { - return new NodeAgentContextImpl( - hostname, - Optional.ofNullable(nodeType).orElse(NodeType.tenant), - Optional.ofNullable(identity).orElseGet(() -> new AthenzService("domain", "service")), - Optional.ofNullable(dockerNetworking).orElse(DockerNetworking.HOST_NETWORK), - Optional.ofNullable(zoneId).orElseGet(() -> new ZoneId(SystemName.dev, Environment.dev, RegionName.defaultName())), - Optional.ofNullable(pathToContainerStorage).orElseGet(() -> Paths.get("/home/docker")), - Optional.ofNullable(pathToVespaHome).orElseGet(() -> Paths.get("/opt/vespa")) - ); - } - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 395b4d458a2..ef2c6ad2800 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -22,7 +22,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.provision.Node; @@ -72,10 +72,11 @@ public class NodeAgentImpl implements NodeAgent { private final Orchestrator orchestrator; private final DockerOperations dockerOperations; private final StorageMaintainer storageMaintainer; + private final Runnable aclMaintainer; + private final Environment environment; private final Clock clock; private final Duration timeBetweenEachConverge; private final Optional<AthenzCredentialsMaintainer> athenzCredentialsMaintainer; - private final Optional<AclMaintainer> aclMaintainer; private int numberOfUnhandledException = 0; private Instant lastConverge; @@ -115,20 +116,22 @@ public class NodeAgentImpl implements NodeAgent { final Orchestrator orchestrator, final DockerOperations dockerOperations, final StorageMaintainer storageMaintainer, + final Runnable aclMaintainer, + final Environment environment, final Clock clock, final Duration timeBetweenEachConverge, - final Optional<AthenzCredentialsMaintainer> athenzCredentialsMaintainer, - final Optional<AclMaintainer> aclMaintainer) { + final Optional<AthenzCredentialsMaintainer> athenzCredentialsMaintainer) { this.context = context; this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; this.dockerOperations = dockerOperations; this.storageMaintainer = storageMaintainer; + this.aclMaintainer = aclMaintainer; + this.environment = environment; this.clock = clock; this.timeBetweenEachConverge = timeBetweenEachConverge; this.lastConverge = clock.instant(); this.athenzCredentialsMaintainer = athenzCredentialsMaintainer; - this.aclMaintainer = aclMaintainer; this.loopThread = new Thread(() -> { try { @@ -225,6 +228,7 @@ public class NodeAgentImpl implements NodeAgent { if (!hasResumedNode) { if (!currentFilebeatRestarter.isPresent()) { storageMaintainer.writeMetricsConfig(context, node); + storageMaintainer.writeFilebeatConfig(context, node); currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay( () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS)); } @@ -505,7 +509,7 @@ public class NodeAgentImpl implements NodeAgent { containerState = STARTING; startContainer(node); containerState = UNKNOWN; - aclMaintainer.ifPresent(AclMaintainer::converge); + aclMaintainer.run(); } verifyHealth(node); @@ -602,8 +606,8 @@ public class NodeAgentImpl implements NodeAgent { Dimensions.Builder dimensionsBuilder = new Dimensions.Builder() .add("host", context.hostname().value()) .add("role", "tenants") - .add("state", node.getState().toString()); - node.getParentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent)); + .add("state", node.getState().toString()) + .add("parentHostname", environment.getParentHostHostname()); node.getAllowedToBeDown().ifPresent(allowed -> dimensionsBuilder.add("orchestratorState", allowed ? "ALLOWED_TO_BE_DOWN" : "NO_REMARKS")); Dimensions dimensions = dimensionsBuilder.build(); |