diff options
Diffstat (limited to 'node-admin/src/main/java/com/yahoo')
15 files changed, 599 insertions, 77 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/AdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/AdminComponent.java new file mode 100644 index 00000000000..9bed492bd76 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/AdminComponent.java @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.component; + +/** + * An AdminComponent cannot assume anything about the environment until enable() + * is called: Required YUM packages may not have been installed, services + * not started, etc. An enabled AdminComponent can be disabled to disengage from + * the environment. + */ +public interface AdminComponent { + /** + * Enable component. May be called more than once. + */ + void enable(); + + /** + * Disable component. May be called more than once. + * Must be compatible with component deconstruct(). + */ + void disable(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java new file mode 100644 index 00000000000..6863a0050cd --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.component; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystem.java new file mode 100644 index 00000000000..c5c2df9e38e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystem.java @@ -0,0 +1,120 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.io; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.nio.file.attribute.UserPrincipal; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.util.Set; + +/** + * File system operations to be mocked in unit tests. + */ +public class FileSystem { + public FileSystemPath withPath(Path path) { + return new FileSystemPath(this, path); + } + + public boolean isDirectory(Path path) { + return path.toFile().isDirectory(); + } + + public boolean isRegularFile(Path path) { + return path.toFile().isFile(); + } + + public void createDirectory(Path path, FileAttribute<?>... attributes) { + uncheck(() -> Files.createDirectory(path, attributes)); + } + + public String readUtf8File(Path path) { + byte[] byteContent = uncheck(() -> Files.readAllBytes(path)); + return new String(byteContent, StandardCharsets.UTF_8); + } + + public void writeUtf8File(Path path, String content, OpenOption... options) { + byte[] contentInUtf8 = content.getBytes(StandardCharsets.UTF_8); + uncheck(() -> Files.write(path, contentInUtf8, options)); + } + + private PosixFileAttributes getAttributes(Path path) { + return uncheck(() -> + Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes()); + } + + public String getPermissions(Path path) { + return PosixFilePermissions.toString(getAttributes(path).permissions()); + } + + /** + * @param permissions Example: "rwxr-x---" means rwx for owner, rx for group, + * and no permissions for others. + */ + public void setPermissions(Path path, String permissions) { + Set<PosixFilePermission> permissionSet; + try { + permissionSet = PosixFilePermissions.fromString(permissions); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Failed to set permissions '" + + permissions + "' on path " + path, e); + } + + uncheck(() -> Files.setPosixFilePermissions(path, permissionSet)); + } + + public String getOwner(Path path) { + return getAttributes(path).owner().getName(); + } + + public void setOwner(Path path, String owner) { + UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService(); + UserPrincipal principal = uncheck(() -> service.lookupPrincipalByName(owner)); + uncheck(() -> Files.setOwner(path, principal)); + } + + public String getGroup(Path path) { + return getAttributes(path).group().getName(); + } + + public void setGroup(Path path, String group) { + UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService(); + GroupPrincipal principal = uncheck(() -> service.lookupPrincipalByGroupName(group)); + uncheck(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class).setGroup(principal)); + } + + @FunctionalInterface + private interface SupplierThrowingIOException<T> { + T get() throws IOException; + } + + private static <T> T uncheck(SupplierThrowingIOException<T> supplier) { + try { + return supplier.get(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @FunctionalInterface + private interface RunnableThrowingIOException<T> { + void run() throws IOException; + } + + private static <T> void uncheck(RunnableThrowingIOException<T> runnable) { + try { + runnable.run(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystemPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystemPath.java new file mode 100644 index 00000000000..bfec341e05c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystemPath.java @@ -0,0 +1,68 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.io; + +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; + +/** + * Convenience class for calling FileSystem methods on a fixed Path. + */ +public class FileSystemPath { + private final FileSystem fileSystem; + private final Path path; + + FileSystemPath(FileSystem fileSystem, Path path) { + this.fileSystem = fileSystem; + this.path = path; + } + + public boolean isDirectory() { + return fileSystem.isDirectory(path); + } + + public boolean isRegularFile() { + return fileSystem.isRegularFile(path); + } + + public FileSystemPath createDirectory(FileAttribute<?>... attributes) { + fileSystem.createDirectory(path, attributes); + return this; + } + + public String readUtf8File() { + return fileSystem.readUtf8File(path); + } + + public FileSystemPath writeUtf8File(String content, OpenOption... options) { + fileSystem.writeUtf8File(path, content, options); + return this; + } + + public String getPermissions() { + return fileSystem.getPermissions(path); + } + + public FileSystemPath setPermissions(String permissions) { + fileSystem.setPermissions(path, permissions); + return this; + } + + public String getOwner() { + return fileSystem.getOwner(path); + } + + public FileSystemPath setOwner(String owner) { + fileSystem.setOwner(path, owner); + return this; + } + + public String getGroup() { + return fileSystem.getGroup(path); + } + + public FileSystemPath setGroup(String group) { + fileSystem.setGroup(path, group); + return this; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java new file mode 100644 index 00000000000..5d8fed67682 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java @@ -0,0 +1,133 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeadmin; + +import com.yahoo.concurrent.classlock.ClassLocking; +import com.yahoo.net.HostName; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.node.admin.component.AdminComponent; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; +import java.util.function.Function; + +/** + * Component that manages Docker containers based on some node repository. + */ +public class DockerAdminComponent implements AdminComponent { + private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); + private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); + + private final NodeAdminConfig config; + private final Docker docker; + private final MetricReceiverWrapper metricReceiver; + private final ClassLocking classLocking; + + private Optional<NodeAdminStateUpdater> nodeAdminStateUpdater = Optional.empty(); + + public DockerAdminComponent(NodeAdminConfig config, + Docker docker, + MetricReceiverWrapper metricReceiver, + ClassLocking classLocking) { + this.config = config; + this.docker = docker; + this.metricReceiver = metricReceiver; + this.classLocking = classLocking; + } + + @Override + public void enable() { + if (nodeAdminStateUpdater.isPresent()) { + return; + } + + + Environment environment = new Environment(); + ConfigServerHttpRequestExecutor requestExecutor = + ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris()); + NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); + Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); + + Clock clock = Clock.systemUTC(); + String dockerHostHostName = HostName.getLocalhost(); + ProcessExecuter processExecuter = new ProcessExecuter(); + + docker.start(); + DockerOperations dockerOperations = new DockerOperationsImpl( + docker, + environment, + processExecuter); + + StorageMaintainer storageMaintainer = new StorageMaintainer( + dockerOperations, + processExecuter, + metricReceiver, + environment, + clock); + + AclMaintainer aclMaintainer = new AclMaintainer( + dockerOperations, + nodeRepository, + dockerHostHostName); + + Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl( + hostName, + nodeRepository, + orchestrator, + dockerOperations, + storageMaintainer, + aclMaintainer, + environment, + clock, + NODE_AGENT_SCAN_INTERVAL); + + NodeAdmin nodeAdmin = new NodeAdminImpl( + dockerOperations, + nodeAgentFactory, + storageMaintainer, + aclMaintainer, + metricReceiver, + clock); + + nodeAdminStateUpdater = Optional.of(new NodeAdminStateUpdater( + nodeRepository, + orchestrator, + storageMaintainer, + nodeAdmin, + dockerHostHostName, + clock, + NODE_ADMIN_CONVERGE_STATE_INTERVAL, + classLocking)); + + nodeAdminStateUpdater.get().start(); + } + + @Override + public void disable() { + if (!nodeAdminStateUpdater.isPresent()) { + return; + } + + nodeAdminStateUpdater.get().stop(); + nodeAdminStateUpdater = Optional.empty(); + // TODO: Also stop docker + } + + public NodeAdminStateUpdater getNodeAdminStateUpdater() { + return nodeAdminStateUpdater.get(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminConfig.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminConfig.java new file mode 100644 index 00000000000..d1aaefab3dd --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminConfig.java @@ -0,0 +1,9 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.nodeadmin; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DockerAdminConfig { +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java index 9caf1307aa4..d9726c88e4c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java @@ -14,14 +14,18 @@ public class NodeAdminConfig { private static final Logger logger = Logger.getLogger(NodeAdminConfig.class.getName()); private static final ObjectMapper mapper = new ObjectMapper(); - enum Mode { + public enum Mode { + aws_tenant, + config_server_host, tenant, - config_server_host } @JsonProperty("mode") public Mode mode = Mode.tenant; + @JsonProperty("docker") + public DockerAdminConfig docker = new DockerAdminConfig(); + public static NodeAdminConfig fromFile(File file) { if (!file.exists()) { return new NodeAdminConfig(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 868ebf39f70..722b1f5a40d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; @@ -47,7 +46,7 @@ public class NodeAdminImpl implements NodeAdmin { private final DockerOperations dockerOperations; private final Function<String, NodeAgent> nodeAgentFactory; private final StorageMaintainer storageMaintainer; - private final AclMaintainer aclMaintainer; + private final Runnable aclMaintainer; private final Clock clock; private boolean previousWantFrozen; @@ -62,7 +61,7 @@ public class NodeAdminImpl implements NodeAdmin { public NodeAdminImpl(final DockerOperations dockerOperations, final Function<String, NodeAgent> nodeAgentFactory, final StorageMaintainer storageMaintainer, - final AclMaintainer aclMaintainer, + final Runnable aclMaintainer, final MetricReceiverWrapper metricReceiver, final Clock clock) { this.dockerOperations = dockerOperations; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java index 1ce6c9a14e1..f5f2deb8dcb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java @@ -2,29 +2,12 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.concurrent.classlock.ClassLocking; -import com.yahoo.net.HostName; -import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.Environment; import java.io.File; -import java.time.Clock; -import java.time.Duration; import java.util.Optional; -import java.util.function.Function; /** * NodeAdminMain is the main component of the node admin JDisc application: @@ -36,16 +19,15 @@ import java.util.function.Function; * be fatal: the node admin may not have installed and started the docker daemon. */ public class NodeAdminMain implements AutoCloseable { - private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); - private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); - private final Docker docker; private final MetricReceiverWrapper metricReceiver; private final ClassLocking classLocking; - private Optional<NodeAdminStateUpdater> nodeAdminStateUpdater = Optional.empty(); + private Optional<DockerAdminComponent> dockerAdmin = Optional.empty(); - public NodeAdminMain(Docker docker, MetricReceiverWrapper metricReceiver, ClassLocking classLocking) { + public NodeAdminMain(Docker docker, + MetricReceiverWrapper metricReceiver, + ClassLocking classLocking) { this.docker = docker; this.metricReceiver = metricReceiver; this.classLocking = classLocking; @@ -53,11 +35,11 @@ public class NodeAdminMain implements AutoCloseable { @Override public void close() { - nodeAdminStateUpdater.ifPresent(NodeAdminStateUpdater::stop); + dockerAdmin.ifPresent(DockerAdminComponent::disable); } public NodeAdminStateUpdater getNodeAdminStateUpdater() { - return nodeAdminStateUpdater.get(); + return dockerAdmin.get().getNodeAdminStateUpdater(); } public void start() { @@ -65,54 +47,24 @@ public class NodeAdminMain implements AutoCloseable { NodeAdminConfig config = NodeAdminConfig.fromFile(new File(staticConfigPath)); switch (config.mode) { + case aws_tenant: case tenant: - setupTenantHostNodeAdmin(); - break; + dockerAdmin = Optional.of(new DockerAdminComponent( + config, + docker, + metricReceiver, + classLocking)); + dockerAdmin.get().enable(); + return; case config_server_host: - setupConfigServerHostNodeAdmin(); - break; - default: - throw new IllegalStateException( - "Unknown bootstrap mode: " + config.mode.name()); + // TODO: + // - install and start docker daemon + // - Read config that specifies which containers to start how + // - use thin static backends for node repo and orchestrator + // - Start node admin state updater. + return; } - } - - private void setupTenantHostNodeAdmin() { - nodeAdminStateUpdater = Optional.of(createNodeAdminStateUpdater()); - nodeAdminStateUpdater.get().start(); - } - - private NodeAdminStateUpdater createNodeAdminStateUpdater() { - Clock clock = Clock.systemUTC(); - String dockerHostHostName = HostName.getLocalhost(); - ProcessExecuter processExecuter = new ProcessExecuter(); - Environment environment = new Environment(); - - ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris()); - NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); - Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); - - docker.start(); - DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter); - - StorageMaintainer storageMaintainer = new StorageMaintainer(dockerOperations, processExecuter, metricReceiver, environment, clock); - AclMaintainer aclMaintainer = new AclMaintainer(dockerOperations, nodeRepository, dockerHostHostName); - - Function<String, NodeAgent> nodeAgentFactory = - (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, - metricReceiver, clock); - - return new NodeAdminStateUpdater(nodeRepository, orchestrator, storageMaintainer, nodeAdmin, - dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, classLocking); - } - private void setupConfigServerHostNodeAdmin() { - // TODO: - // - install and start docker daemon - // - Read config that specifies which containers to start how - // - use thin static backends for node repo, orchestrator, and others - // - Start core node admin. + throw new IllegalStateException("Unknown bootstrap mode: " + config.mode.name()); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index b8a8bbfd6ad..8ca3ace92e5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; @@ -71,7 +70,7 @@ public class NodeAgentImpl implements NodeAgent { private final Orchestrator orchestrator; private final DockerOperations dockerOperations; private final StorageMaintainer storageMaintainer; - private final AclMaintainer aclMaintainer; + private final Runnable aclMaintainer; private final Environment environment; private final Clock clock; private final Duration timeBetweenEachConverge; @@ -116,7 +115,7 @@ public class NodeAgentImpl implements NodeAgent { final Orchestrator orchestrator, final DockerOperations dockerOperations, final StorageMaintainer storageMaintainer, - final AclMaintainer aclMaintainer, + final Runnable aclMaintainer, final Environment environment, final Clock clock, final Duration timeBetweenEachConverge) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/AddYumRepoTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/AddYumRepoTask.java new file mode 100644 index 00000000000..a2ed6a80084 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/AddYumRepoTask.java @@ -0,0 +1,61 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.regex.Pattern; + +public class AddYumRepoTask implements Task { + private static final Pattern REPOSITORY_ID_PATTERN = Pattern.compile("^[a-zA-Z_-]+$"); + + private final String repositoryId; // e.g. "platform_rpms-latest" + private final String name; // e.g. "Platform RPM Latest Repo" + private final String baseurl; + private final boolean enabled; + + public AddYumRepoTask(String repositoryId, + String name, + String baseurl, + boolean enabled) { + this.repositoryId = repositoryId; + this.name = name; + this.baseurl = baseurl; + this.enabled = enabled; + validateRepositoryId(repositoryId); + } + + @Override + public boolean execute(TaskContext context) { + Path path = Paths.get("/etc/yum.repos.d",repositoryId + ".repo"); + + if (context.getFileSystem().isRegularFile(path)) { + return false; + } + + WriteFileTask writeFileTask = new WriteFileTask(path, this::getRepoFileContent) + .withOwner("root") + .withGroup("root") + .withPermissions("rw-r--r--"); + + return context.executeSubtask(writeFileTask); + } + + String getRepoFileContent() { + return String.join("\n", + "# This file was generated by node admin", + "# Do NOT modify this file by hand", + "", + "[" + repositoryId + "]", + "name=" + name, + "baseurl=" + baseurl, + "enabled=" + (enabled ? 1 : 0), + "gpgcheck=0" + ) + "\n"; + } + + static void validateRepositoryId(String repositoryId) { + if (!REPOSITORY_ID_PATTERN.matcher(repositoryId).matches()) { + throw new IllegalArgumentException("Invalid repository ID '" + repositoryId + "'"); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/MakeDirectoryTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/MakeDirectoryTask.java new file mode 100644 index 00000000000..522abb81248 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/MakeDirectoryTask.java @@ -0,0 +1,49 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task; + +import com.yahoo.vespa.hosted.node.admin.io.FileSystem; + +import java.nio.file.Path; + +public class MakeDirectoryTask implements Task { + private final Path path; + private boolean withParents = false; + + public MakeDirectoryTask(Path path) { + this.path = path; + } + + public MakeDirectoryTask withParents() { + this.withParents = true; + return this; + } + + Path getPath() { + return path; + } + + boolean getWithParents() { + return withParents; + } + + private boolean makeDirectory(FileSystem fileSystem, + Path directory, + boolean withParents) { + if (fileSystem.isDirectory(directory)) { + return false; + } + + if (withParents) { + makeDirectory(fileSystem, directory.getParent(), withParents); + } + + fileSystem.createDirectory(directory); + + return true; + } + + @Override + public boolean execute(TaskContext context) { + return makeDirectory(context.getFileSystem(), path, withParents); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/Task.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/Task.java new file mode 100644 index 00000000000..4d40ac02440 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/Task.java @@ -0,0 +1,17 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task; + +import com.yahoo.vespa.hosted.node.admin.io.FileSystem; + +public interface Task { + interface TaskContext { + FileSystem getFileSystem(); + boolean executeSubtask(Task task); + } + + /** + * @return Returns false if task was a no-op. Used for informational purposes only. + * @throws RuntimeException if task could not be completed. + */ + boolean execute(TaskContext context); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/WriteFileTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/WriteFileTask.java new file mode 100644 index 00000000000..308a7470d24 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/WriteFileTask.java @@ -0,0 +1,80 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task; + +import com.yahoo.vespa.hosted.node.admin.io.FileSystemPath; +import org.glassfish.jersey.internal.util.Producer; + +import java.nio.file.Path; +import java.util.Optional; + +public class WriteFileTask implements Task { + private final Path path; + private final Producer<String> contentProducer; + + private Optional<String> owner = Optional.empty(); + private Optional<String> group = Optional.empty(); + private Optional<String> permissions = Optional.empty(); + + public WriteFileTask(Path path, Producer<String> contentProducer) { + this.path = path; + this.contentProducer = contentProducer; + } + + public WriteFileTask withOwner(String owner) { + this.owner = Optional.of(owner); + return this; + } + + public WriteFileTask withGroup(String group) { + this.group = Optional.of(group); + return this; + } + + /** + * @param permissions of the form "rwxr-x---". + */ + public WriteFileTask withPermissions(String permissions) { + this.permissions = Optional.of(permissions); + return this; + } + + @Override + public boolean execute(TaskContext context) { + final FileSystemPath fileSystemPath = context.getFileSystem().withPath(path); + + // TODO: Only return false if content, permission, etc would be unchanged. + if (fileSystemPath.isRegularFile()) { + return false; + } + + context.executeSubtask(new MakeDirectoryTask(path.getParent()).withParents()); + + String content = contentProducer.call(); + fileSystemPath.writeUtf8File(content); + permissions.ifPresent(fileSystemPath::setPermissions); + owner.ifPresent(fileSystemPath::setOwner); + group.ifPresent(fileSystemPath::setGroup); + + return true; + } + + public Path getPath() { + return path; + } + + public Producer<String> getContentProducer() { + return contentProducer; + } + + public Optional<String> getOwner() { + return owner; + } + + public Optional<String> getGroup() { + return group; + } + + public Optional<String> getPermissions() { + return permissions; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/package-info.java new file mode 100644 index 00000000000..433884139f6 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.task; + +import com.yahoo.osgi.annotation.ExportPackage; |