diff options
Diffstat (limited to 'node-admin/src/main/java/com')
5 files changed, 99 insertions, 154 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java index 27dc4e5237b..cedddc9b0b6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.docker; import com.yahoo.vespa.hosted.dockerapi.Container; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.DockerImage; @@ -31,7 +30,7 @@ public interface DockerOperations { ProcessResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command); - ProcessResult executeCommandInNetworkNamespace(ContainerName containerName, String... command); + ProcessResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command); /** Resume node. Resuming a node means that it is ready to take on traffic. */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index bdda155b2f3..d65bca89733 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -6,7 +6,6 @@ import com.yahoo.collections.Pair; import com.yahoo.config.provision.NodeType; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Container; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.Docker; @@ -196,12 +195,12 @@ public class DockerOperationsImpl implements DockerOperations { } @Override - public ProcessResult executeCommandInNetworkNamespace(ContainerName containerName, String... command) { - final Integer containerPid = docker.getContainer(containerName) + public ProcessResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) { + final int containerPid = docker.getContainer(context.containerName()) .filter(container -> container.state.isRunning()) - .map(container -> container.pid) - .orElseThrow(() -> new RuntimeException("PID not found for container with name: " + - containerName.asString())); + .orElseThrow(() -> new RuntimeException( + "Found no running container named " + context.containerName().asString())) + .pid; final String[] wrappedCommand = Stream.concat( Stream.of("nsenter", String.format("--net=/proc/%d/ns/net", containerPid), "--"), @@ -213,12 +212,12 @@ public class DockerOperationsImpl implements DockerOperations { if (result.getFirst() != 0) { throw new RuntimeException(String.format( "Failed to execute %s in network namespace for %s (PID = %d), exit code: %d, output: %s", - Arrays.toString(wrappedCommand), containerName.asString(), containerPid, result.getFirst(), result.getSecond())); + Arrays.toString(wrappedCommand), context.containerName().asString(), containerPid, result.getFirst(), result.getSecond())); } return new ProcessResult(0, result.getSecond(), ""); } catch (IOException e) { throw new RuntimeException(String.format("IOException while executing %s in network namespace for %s (PID = %d)", - Arrays.toString(wrappedCommand), containerName.asString(), containerPid), e); + Arrays.toString(wrappedCommand), context.containerName().asString(), containerPid), e); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index a733a52ea10..a68eda7e39e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -2,19 +2,29 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.acl; import com.google.common.net.InetAddresses; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.dockerapi.Container; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; +import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; +import java.io.IOException; import java.net.InetAddress; -import java.util.Map; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.yolean.Exceptions.uncheck; + /** * This class maintains the iptables (ipv4 and ipv6) for all running containers. * The filter table is synced with ACLs fetched from the Node repository while the nat table @@ -31,56 +41,100 @@ import java.util.stream.Collectors; * @author smorgrav */ public class AclMaintainer { - - private static final PrefixLogger log = PrefixLogger.getNodeAdminLogger(AclMaintainer.class); + private static final Logger logger = Logger.getLogger(AclMaintainer.class.getName()); private final DockerOperations dockerOperations; - private final NodeRepository nodeRepository; private final IPAddresses ipAddresses; - private final String hostHostname; - public AclMaintainer(DockerOperations dockerOperations, NodeRepository nodeRepository, - HostName hostHostname, IPAddresses ipAddresses) { + public AclMaintainer(DockerOperations dockerOperations, IPAddresses ipAddresses) { this.dockerOperations = dockerOperations; - this.nodeRepository = nodeRepository; this.ipAddresses = ipAddresses; - this.hostHostname = hostHostname.value(); } - private void applyRedirect(Container container, InetAddress address) { + public void converge(NodeAgentContext context) { + // Apply acl to the filter table + editFlushOnError(context, IPVersion.IPv4, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv4)); + editFlushOnError(context, IPVersion.IPv6, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv6)); + + ipAddresses.getAddress(context.hostname().value(), IPVersion.IPv4).ifPresent(addr -> applyRedirect(context, addr)); + ipAddresses.getAddress(context.hostname().value(), IPVersion.IPv6).ifPresent(addr -> applyRedirect(context, addr)); + } + + private void applyRedirect(NodeAgentContext context, InetAddress address) { IPVersion ipVersion = IPVersion.get(address); // Necessary to avoid the routing packets destined for the node's own public IP address // via the bridge, which is illegal. String redirectRule = "-A OUTPUT -d " + InetAddresses.toAddrString(address) + ipVersion.singleHostCidr() + " -j REDIRECT"; - IPTablesEditor.editLogOnError(dockerOperations, container.name, ipVersion, "nat", NatTableLineEditor.from(redirectRule)); + editLogOnError(context, ipVersion, "nat", NatTableLineEditor.from(redirectRule)); } - private void apply(Container container, Acl acl) { - // Apply acl to the filter table - IPTablesEditor.editFlushOnError(dockerOperations, container.name, IPVersion.IPv6, "filter", FilterTableLineEditor.from(acl, IPVersion.IPv6)); - IPTablesEditor.editFlushOnError(dockerOperations, container.name, IPVersion.IPv4, "filter", FilterTableLineEditor.from(acl, IPVersion.IPv4)); + private boolean editFlushOnError(NodeAgentContext context, IPVersion ipVersion, String table, LineEditor lineEditor) { + return edit(context, table, ipVersion, lineEditor, true); + } + + private boolean editLogOnError(NodeAgentContext context, IPVersion ipVersion, String table, LineEditor lineEditor) { + return edit(context, table, ipVersion, lineEditor, false); + } - ipAddresses.getAddress(container.hostname, IPVersion.IPv4).ifPresent(addr -> applyRedirect(container, addr)); - ipAddresses.getAddress(container.hostname, IPVersion.IPv6).ifPresent(addr -> applyRedirect(container, addr)); + private boolean edit(NodeAgentContext context, String table, IPVersion ipVersion, LineEditor lineEditor, boolean flush) { + Editor editor = new Editor( + ipVersion.iptablesCmd() + "-" + table, + listTable(context, table, ipVersion), + restoreTable(context, table, ipVersion, flush), + lineEditor); + return editor.edit(message -> context.log(logger, message)); } - private synchronized void configureAcls() { - log.info("Configuring ACLs"); // Needed to potentially nail down when ACL maintainer stopped working - Map<String, Container> runningContainers = dockerOperations - .getAllManagedContainers().stream() - .filter(container -> container.state.isRunning()) - .collect(Collectors.toMap(container -> container.hostname, container -> container)); + private Supplier<List<String>> listTable(NodeAgentContext context, String table, IPVersion ipVersion) { + return () -> { + ProcessResult currentRulesResult = + dockerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesCmd(), "-S", "-t", table); + return Arrays.stream(currentRulesResult.getOutput().split("\n")) + .map(String::trim) + .collect(Collectors.toList()); + }; + } - nodeRepository.getAcls(hostHostname).entrySet().stream() - .filter(entry -> runningContainers.containsKey(entry.getKey())) - .forEach(entry -> apply(runningContainers.get(entry.getKey()), entry.getValue())); + private Consumer<List<String>> restoreTable(NodeAgentContext context, String table, IPVersion ipVersion, boolean flush) { + return list -> { + try (TemporaryIpTablesFileHandler fileHandler = new TemporaryIpTablesFileHandler(table)) { + String rules = String.join("\n", list); + String fileContent = "*" + table + "\n" + rules + "\nCOMMIT\n"; + fileHandler.writeUtf8Content(fileContent); + dockerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesRestore(), fileHandler.absolutePath()); + } catch (Exception e) { + if (flush) { + context.log(logger, LogLevel.ERROR, "Exception occurred while syncing iptable " + table + ", attempting rollback", e); + try { + dockerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesCmd(), "-F", "-t", table); + } catch (Exception ne) { + context.log(logger, LogLevel.ERROR, "Rollback of table " + table + " failed, giving up", ne); + } + } else { + context.log(logger, LogLevel.WARNING, "Unable to sync iptables for " + table, e); + } + } + }; } - public void converge() { - try { - configureAcls(); - } catch (Throwable t) { - log.error("Failed to configure ACLs", t); + private static class TemporaryIpTablesFileHandler implements AutoCloseable { + private final Path path; + + private TemporaryIpTablesFileHandler(String table) { + this.path = uncheck(() -> Files.createTempFile("iptables-restore", "." + table)); + } + + private void writeUtf8Content(String content) throws IOException { + Files.write(path, content.getBytes(StandardCharsets.UTF_8)); + } + + private String absolutePath() { + return path.toAbsolutePath().toString(); + } + + @Override + public void close() throws IOException { + Files.deleteIfExists(path); } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditor.java deleted file mode 100644 index d34b41e3ae5..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditor.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; -import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -/** - * Edit the iptables for docker containers. - */ -class IPTablesEditor { - - private final PrefixLogger log; - private final DockerOperations dockerOperations; - private final ContainerName containerName; - private final Consumer<String> testInterceptor; - - public IPTablesEditor(DockerOperations dockerOperations, ContainerName containerName) { - this(dockerOperations, containerName, (result) -> {}); - } - - IPTablesEditor(DockerOperations dockerOperations, ContainerName containerName, Consumer<String> testInterceptor) { - this.dockerOperations = dockerOperations; - this.containerName = containerName; - this.testInterceptor = testInterceptor; - this.log = PrefixLogger.getNodeAgentLogger(AclMaintainer.class, containerName); - } - - public static boolean editFlushOnError(DockerOperations dockerOperations, ContainerName containerName, IPVersion ipVersion, String table, LineEditor lineEditor) { - return new IPTablesEditor(dockerOperations, containerName).edit(table, ipVersion, lineEditor, true); - } - - public static boolean editLogOnError(DockerOperations dockerOperations, ContainerName containerName, IPVersion ipVersion, String table, LineEditor lineEditor) { - return new IPTablesEditor(dockerOperations, containerName).edit(table, ipVersion, lineEditor, false); - } - - public boolean edit(String table, IPVersion ipVersion, LineEditor lineEditor, boolean flush) { - String editId = ipVersion.iptablesCmd() + "-" + table; - Editor editor = new Editor(editId, listTable(table, ipVersion), restoreTable(table, ipVersion, flush), lineEditor); - return editor.edit(log::info); - } - - private Supplier<List<String>> listTable(String table, IPVersion ipVersion) { - return () -> { - ProcessResult currentRulesResult = - dockerOperations.executeCommandInNetworkNamespace(containerName, ipVersion.iptablesCmd(), "-S", "-t", table); - return Arrays.stream(currentRulesResult.getOutput().split("\n")) - .map(String::trim) - .collect(Collectors.toList()); - }; - } - - private Consumer<List<String>> restoreTable(String table, IPVersion ipVersion, boolean flush) { - return list -> { - File file = null; - try { - String rules = String.join("\n", list); - String fileContent = "*" + table + "\n" + rules + "\nCOMMIT\n"; - file = writeTempFile(table, fileContent); - dockerOperations.executeCommandInNetworkNamespace(containerName, ipVersion.iptablesRestore(), file.getAbsolutePath()); - testInterceptor.accept(fileContent); - } catch (Exception e) { - if (flush) { - log.error("Exception occurred while syncing iptable " + table + " for " + containerName.asString() + ", attempting rollback", e); - try { - dockerOperations.executeCommandInNetworkNamespace(containerName, ipVersion.iptablesCmd(), "-F", "-t", table); - } catch (Exception ne) { - log.error("Rollback of table " + table + " for " + containerName.asString() + " failed, giving up", ne); - } - } else { - log.warning("Unable to sync iptables for " + table, e); - } - } finally { - if (file != null) { - file.delete(); - } - } - }; - } - - private File writeTempFile(String table, String content) { - try { - Path path = Files.createTempFile("iptables-restore", "." + table); - File file = path.toFile(); - Files.write(path, content.getBytes(StandardCharsets.UTF_8)); - file.deleteOnExit(); - return file; - } catch (IOException e) { - throw new RuntimeException("Unable to write restore file for iptables.", e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index e5e045c6013..d31a888ea44 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -468,11 +468,11 @@ public class NodeAgentImpl implements NodeAgent { containerState = STARTING; startContainer(context); containerState = UNKNOWN; - aclMaintainer.ifPresent(AclMaintainer::converge); } else { updateContainerIfNeeded(context, container.get()); } + aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); startServicesIfNeeded(context); resumeNodeIfNeeded(context); healthChecker.ifPresent(checker -> checker.verifyHealth(context)); |