diff options
author | Håkon Hallingstad <hakon@oath.com> | 2018-01-05 16:54:37 +0100 |
---|---|---|
committer | Håkon Hallingstad <hakon@oath.com> | 2018-01-05 16:54:37 +0100 |
commit | 49a6ff5aa3296a99d10334856fdfc443bb844805 (patch) | |
tree | 15c333f59c827f0a721d4c55ebe9f56634167220 /node-admin | |
parent | f5df837564f3e60216657cc8da35849887539c18 (diff) |
Introduce Task and AdminComponent
Diffstat (limited to 'node-admin')
15 files changed, 594 insertions, 93 deletions
diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index 030c42ac8c5..27de4c856e8 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -1,23 +1,27 @@ <?xml version="1.0" encoding="utf-8" ?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<jdisc id="node-admin" jetty="true" version="1.0"> - <!-- Please update container test when changing this file --> - <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> - <handler id="com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler" bundle="node-admin"> - <binding>http://*/rest/*</binding> - </handler> - <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.NodeAdminProvider" bundle="node-admin"/> - <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> - <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> +<service version="1.0" xmlns:preprocess="properties"> + <jdisc id="node-admin" jetty="true" version="1.0"> + <!-- Please update container test when changing this file --> + <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> + <handler id="com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler" bundle="node-admin"> + <binding>http://*/rest/*</binding> + </handler> + <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.NodeAdminProvider" bundle="node-admin"/> + <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> + <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> - <config name='vespa.hosted.dockerapi.docker'> - <isRunningLocally>false</isRunningLocally> - </config> - - <config name='vespa.hosted.node.admin.node-admin'> - <isRunningLocally>false</isRunningLocally> - <restartOnDeploy>true</restartOnDeploy> - </config> - - <nodes type="host"/> -</jdisc> + <config name="vespa.hosted.dockerapi.docker"> + <isRunningLocally>false</isRunningLocally> + </config> + + <config name="vespa.hosted.node.admin.node-admin"> + <isRunningLocally>false</isRunningLocally> + <restartOnDeploy>true</restartOnDeploy> + </config> + + <nodes type="host"/> + + <preprocess:include file="variant.xml" required="false"/> + </jdisc> +</service> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/AdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/AdminComponent.java new file mode 100644 index 00000000000..9bed492bd76 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/AdminComponent.java @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.component; + +/** + * An AdminComponent cannot assume anything about the environment until enable() + * is called: Required YUM packages may not have been installed, services + * not started, etc. An enabled AdminComponent can be disabled to disengage from + * the environment. + */ +public interface AdminComponent { + /** + * Enable component. May be called more than once. + */ + void enable(); + + /** + * Disable component. May be called more than once. + * Must be compatible with component deconstruct(). + */ + void disable(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java new file mode 100644 index 00000000000..6863a0050cd --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.component; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystem.java new file mode 100644 index 00000000000..de20f0975f6 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystem.java @@ -0,0 +1,90 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.io; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.nio.file.attribute.UserPrincipal; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.util.Set; + +/** + * File system operations to be mocked in unit tests. + */ +public class FileSystem { + public FileSystemPath withPath(Path path) { + return new FileSystemPath(this, path); + } + + public boolean isDirectory(Path path) { + return path.toFile().isDirectory(); + } + + public boolean isRegularFile(Path path) { + return path.toFile().isFile(); + } + + public void createDirectory(Path path, FileAttribute<?>... attributes) { + uncheck(() -> Files.createDirectory(path, attributes)); + } + + public void writeUtf8File(Path path, String content, OpenOption... options) { + byte[] contentInUtf8 = content.getBytes(StandardCharsets.UTF_8); + uncheck(() -> Files.write(path, contentInUtf8, options)); + } + + /** + * @param permissions Example: "rwxr-x---" means rwx for owner, rx for group, + * and no permissions for others. + */ + public void setPermissions(Path path, String permissions) { + Set<PosixFilePermission> permissionSet = PosixFilePermissions.fromString(permissions); + uncheck(() -> Files.setPosixFilePermissions(path, permissionSet)); + } + + public void setOwner(Path path, String owner) { + UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService(); + UserPrincipal principal = uncheck(() -> service.lookupPrincipalByName(owner)); + uncheck(() -> Files.setOwner(path, principal)); + } + + public void setGroup(Path path, String group) { + UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService(); + GroupPrincipal principal = uncheck(() -> service.lookupPrincipalByGroupName(group)); + uncheck(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class).setGroup(principal)); + } + + @FunctionalInterface + private interface SupplierThrowingIOException<T> { + T get() throws IOException; + } + + private static <T> T uncheck(SupplierThrowingIOException<T> supplier) { + try { + return supplier.get(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @FunctionalInterface + private interface RunnableThrowingIOException<T> { + void run() throws IOException; + } + + private static <T> void uncheck(RunnableThrowingIOException<T> runnable) { + try { + runnable.run(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystemPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystemPath.java new file mode 100644 index 00000000000..6fa32ea8151 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/io/FileSystemPath.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.io; + +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; + +/** + * Convenience class for calling FileSystem methods on a fixed Path. + */ +public class FileSystemPath { + private final FileSystem fileSystem; + private final Path path; + + FileSystemPath(FileSystem fileSystem, Path path) { + this.fileSystem = fileSystem; + this.path = path; + } + + public boolean isDirectory() { + return fileSystem.isDirectory(path); + } + + public boolean isRegularFile() { + return fileSystem.isRegularFile(path); + } + + public FileSystemPath createDirectory(FileAttribute<?>... attributes) { + fileSystem.createDirectory(path, attributes); + return this; + } + + public FileSystemPath writeUtf8File(String content, OpenOption... options) { + fileSystem.writeUtf8File(path, content, options); + return this; + } + + public FileSystemPath setPermissions(String permissions) { + fileSystem.setPermissions(path, permissions); + return this; + } + + public FileSystemPath setOwner(String owner) { + fileSystem.setOwner(path, owner); + return this; + } + + public FileSystemPath setGroup(String group) { + fileSystem.setGroup(path, group); + return this; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java new file mode 100644 index 00000000000..928bf17db6e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java @@ -0,0 +1,128 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeadmin; + +import com.yahoo.concurrent.classlock.ClassLocking; +import com.yahoo.net.HostName; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.node.admin.component.AdminComponent; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; +import java.util.function.Function; + +/** + * Component that manages Docker containers based on some node repository. + */ +public class DockerAdminComponent implements AdminComponent { + private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); + private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); + + private final Environment environment; + private final NodeRepository nodeRepository; + private final Orchestrator orchestrator; + private final Docker docker; + private final MetricReceiverWrapper metricReceiver; + private final ClassLocking classLocking; + private Optional<NodeAdminStateUpdater> nodeAdminStateUpdater = Optional.empty(); + + public DockerAdminComponent(Environment environment, + NodeRepository nodeRepository, + Orchestrator orchestrator, + Docker docker, + MetricReceiverWrapper metricReceiver, + ClassLocking classLocking) { + this.environment = environment; + this.nodeRepository = nodeRepository; + this.orchestrator = orchestrator; + this.docker = docker; + this.metricReceiver = metricReceiver; + this.classLocking = classLocking; + } + + @Override + public void enable() { + if (nodeAdminStateUpdater.isPresent()) { + return; + } + + Clock clock = Clock.systemUTC(); + String dockerHostHostName = HostName.getLocalhost(); + ProcessExecuter processExecuter = new ProcessExecuter(); + + docker.start(); + DockerOperations dockerOperations = new DockerOperationsImpl( + docker, + environment, + processExecuter); + + StorageMaintainer storageMaintainer = new StorageMaintainer( + dockerOperations, + processExecuter, + metricReceiver, + environment, + clock); + + AclMaintainer aclMaintainer = new AclMaintainer( + dockerOperations, + nodeRepository, + dockerHostHostName); + + Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl( + hostName, + nodeRepository, + orchestrator, + dockerOperations, + storageMaintainer, + aclMaintainer, + environment, + clock, + NODE_AGENT_SCAN_INTERVAL); + + NodeAdmin nodeAdmin = new NodeAdminImpl( + dockerOperations, + nodeAgentFactory, + storageMaintainer, + aclMaintainer, + metricReceiver, + clock); + + nodeAdminStateUpdater = Optional.of(new NodeAdminStateUpdater( + nodeRepository, + orchestrator, + storageMaintainer, + nodeAdmin, + dockerHostHostName, + clock, + NODE_ADMIN_CONVERGE_STATE_INTERVAL, + classLocking)); + + nodeAdminStateUpdater.get().start(); + } + + @Override + public void disable() { + if (!nodeAdminStateUpdater.isPresent()) { + return; + } + + nodeAdminStateUpdater.get().stop(); + nodeAdminStateUpdater = Optional.empty(); + // TODO: Also stop docker + } + + public NodeAdminStateUpdater getNodeAdminStateUpdater() { + return nodeAdminStateUpdater.get(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 868ebf39f70..722b1f5a40d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; @@ -47,7 +46,7 @@ public class NodeAdminImpl implements NodeAdmin { private final DockerOperations dockerOperations; private final Function<String, NodeAgent> nodeAgentFactory; private final StorageMaintainer storageMaintainer; - private final AclMaintainer aclMaintainer; + private final Runnable aclMaintainer; private final Clock clock; private boolean previousWantFrozen; @@ -62,7 +61,7 @@ public class NodeAdminImpl implements NodeAdmin { public NodeAdminImpl(final DockerOperations dockerOperations, final Function<String, NodeAgent> nodeAgentFactory, final StorageMaintainer storageMaintainer, - final AclMaintainer aclMaintainer, + final Runnable aclMaintainer, final MetricReceiverWrapper metricReceiver, final Clock clock) { this.dockerOperations = dockerOperations; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java index 1ce6c9a14e1..58358efff33 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java @@ -2,17 +2,9 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.concurrent.classlock.ClassLocking; -import com.yahoo.net.HostName; -import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; @@ -21,10 +13,6 @@ import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; import com.yahoo.vespa.hosted.node.admin.util.Environment; import java.io.File; -import java.time.Clock; -import java.time.Duration; -import java.util.Optional; -import java.util.function.Function; /** * NodeAdminMain is the main component of the node admin JDisc application: @@ -36,28 +24,34 @@ import java.util.function.Function; * be fatal: the node admin may not have installed and started the docker daemon. */ public class NodeAdminMain implements AutoCloseable { - private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); - private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); + private final DockerAdminComponent dockerAdmin; - private final Docker docker; - private final MetricReceiverWrapper metricReceiver; - private final ClassLocking classLocking; + public NodeAdminMain(Docker docker, + MetricReceiverWrapper metricReceiver, + ClassLocking classLocking) { + Environment environment = new Environment(); + ConfigServerHttpRequestExecutor requestExecutor = + ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris()); - private Optional<NodeAdminStateUpdater> nodeAdminStateUpdater = Optional.empty(); + NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); + Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); - public NodeAdminMain(Docker docker, MetricReceiverWrapper metricReceiver, ClassLocking classLocking) { - this.docker = docker; - this.metricReceiver = metricReceiver; - this.classLocking = classLocking; + dockerAdmin = new DockerAdminComponent( + environment, + nodeRepository, + orchestrator, + docker, + metricReceiver, + classLocking); } @Override public void close() { - nodeAdminStateUpdater.ifPresent(NodeAdminStateUpdater::stop); + dockerAdmin.disable(); } public NodeAdminStateUpdater getNodeAdminStateUpdater() { - return nodeAdminStateUpdater.get(); + return dockerAdmin.getNodeAdminStateUpdater(); } public void start() { @@ -66,53 +60,18 @@ public class NodeAdminMain implements AutoCloseable { switch (config.mode) { case tenant: - setupTenantHostNodeAdmin(); + dockerAdmin.enable(); break; case config_server_host: - setupConfigServerHostNodeAdmin(); + // TODO: + // - install and start docker daemon + // - Read config that specifies which containers to start how + // - use thin static backends for node repo and orchestrator + // - Start node admin state updater. break; default: throw new IllegalStateException( "Unknown bootstrap mode: " + config.mode.name()); } } - - private void setupTenantHostNodeAdmin() { - nodeAdminStateUpdater = Optional.of(createNodeAdminStateUpdater()); - nodeAdminStateUpdater.get().start(); - } - - private NodeAdminStateUpdater createNodeAdminStateUpdater() { - Clock clock = Clock.systemUTC(); - String dockerHostHostName = HostName.getLocalhost(); - ProcessExecuter processExecuter = new ProcessExecuter(); - Environment environment = new Environment(); - - ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris()); - NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); - Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); - - docker.start(); - DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter); - - StorageMaintainer storageMaintainer = new StorageMaintainer(dockerOperations, processExecuter, metricReceiver, environment, clock); - AclMaintainer aclMaintainer = new AclMaintainer(dockerOperations, nodeRepository, dockerHostHostName); - - Function<String, NodeAgent> nodeAgentFactory = - (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, - metricReceiver, clock); - - return new NodeAdminStateUpdater(nodeRepository, orchestrator, storageMaintainer, nodeAdmin, - dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, classLocking); - } - - private void setupConfigServerHostNodeAdmin() { - // TODO: - // - install and start docker daemon - // - Read config that specifies which containers to start how - // - use thin static backends for node repo, orchestrator, and others - // - Start core node admin. - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index b8a8bbfd6ad..8ca3ace92e5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; @@ -71,7 +70,7 @@ public class NodeAgentImpl implements NodeAgent { private final Orchestrator orchestrator; private final DockerOperations dockerOperations; private final StorageMaintainer storageMaintainer; - private final AclMaintainer aclMaintainer; + private final Runnable aclMaintainer; private final Environment environment; private final Clock clock; private final Duration timeBetweenEachConverge; @@ -116,7 +115,7 @@ public class NodeAgentImpl implements NodeAgent { final Orchestrator orchestrator, final DockerOperations dockerOperations, final StorageMaintainer storageMaintainer, - final AclMaintainer aclMaintainer, + final Runnable aclMaintainer, final Environment environment, final Clock clock, final Duration timeBetweenEachConverge) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/AddYumRepoTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/AddYumRepoTask.java new file mode 100644 index 00000000000..b3c85b8614e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/AddYumRepoTask.java @@ -0,0 +1,75 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.regex.Pattern; + +public class AddYumRepoTask implements Task { + private static final Pattern REPOSITORY_ID_PATTERN = Pattern.compile("^[a-zA-Z_-]+$"); + + private final Params params; + + public static class Params { + private final String repositoryId; // e.g. "platform_rpms-latest" + private final String name; // e.g. "Platform RPM Latest Repo" + private final String baseurl; + private final boolean enabled; + + public Params(String repositoryId, + String name, + String baseurl, + boolean enabled) { + this.repositoryId = repositoryId; + this.name = name; + this.baseurl = baseurl; + this.enabled = enabled; + } + } + + public AddYumRepoTask(Params params) { + validateRepositoryId(params.repositoryId); + this.params = params; + Task.validateVariant(variantName()); + } + + @Override + public String variantName() { + return params.repositoryId; + } + + @Override + public boolean execute(TaskContext context) { + Path path = Paths.get("/etc/yum.repos.d",params.repositoryId, ".repo"); + + if (path.toFile().exists()) { + return false; + } + + WriteFileTask.Params writeFileParams = new WriteFileTask.Params(path, this::getRepoFileContent) + .withUser("root") + .withGroup("root") + .withPermissions("rw-r--r--"); + + return context.executeSubtask(new WriteFileTask(writeFileParams)); + } + + String getRepoFileContent() { + return String.join("\n", + "# This file was generated by node admin", + "# Do NOT modify this file by hand", + "", + "[" + params.repositoryId + "]", + "name=" + params.name, + "baseurl=" + params.baseurl, + "enabled=" + (params.enabled ? 1 : 0), + "gpgcheck=0" + ) + "\n"; + } + + static void validateRepositoryId(String repositoryId) { + if (!REPOSITORY_ID_PATTERN.matcher(repositoryId).matches()) { + throw new IllegalArgumentException("Invalid repository ID '" + repositoryId + "'"); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/MakeDirectoryTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/MakeDirectoryTask.java new file mode 100644 index 00000000000..c406cd13c22 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/MakeDirectoryTask.java @@ -0,0 +1,47 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task; + +import com.yahoo.vespa.hosted.node.admin.io.FileSystem; + +import java.nio.file.Path; + +public class MakeDirectoryTask implements Task { + private final Path directory; + private boolean withParents = false; + + MakeDirectoryTask(Path directory) { + this.directory = directory; + validate(); + } + + public MakeDirectoryTask withParents() { + this.withParents = true; + return this; + } + + @Override + public String variantName() { + return directory.getFileName().toString(); + } + + private boolean makeDirectory(FileSystem fileSystem, + Path directory, + boolean withParents) { + if (fileSystem.isDirectory(directory)) { + return false; + } + + if (withParents) { + makeDirectory(fileSystem, directory.getParent(), withParents); + } + + fileSystem.createDirectory(directory); + + return true; + } + + @Override + public boolean execute(TaskContext context) { + return !makeDirectory(context.getFileSystem(), directory, withParents); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/Task.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/Task.java new file mode 100644 index 00000000000..c630bbeb3e1 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/Task.java @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task; + +import com.yahoo.vespa.hosted.node.admin.io.FileSystem; + +import java.util.regex.Pattern; + +public interface Task { + /** + * If the task supports different variants, this method should be overridden + * to return something distinctive for this task instance. For instance a + * "WriteFileTask" that supports writing files at a 'path' could return + * the stem of the filename as the variantName(). variantName() must match + * NAME_PATTERN to ensure name is valid as key in various contexts. + */ + default String variantName() { return ""; } + + default void validate() { validateVariant(variantName()); } + static void validateVariant(String variant) { + Pattern variantPattern = Pattern.compile("^[a-zA-Z0-9_-]*$"); + if (!variantPattern.matcher(variant).matches()) { + throw new IllegalArgumentException("variantName '" + variant + "' is not a valid"); + } + } + + interface TaskContext { + FileSystem getFileSystem(); + boolean executeSubtask(Task task); + } + + /** + * @return Returns false if task was a no-op. Used for informational purposes only. + * @throws RuntimeException if task could not be completed. + */ + boolean execute(TaskContext context); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/WriteFileTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/WriteFileTask.java new file mode 100644 index 00000000000..694454e121a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/WriteFileTask.java @@ -0,0 +1,82 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task; + +import com.yahoo.vespa.hosted.node.admin.io.FileSystemPath; +import org.glassfish.jersey.internal.util.Producer; + +import java.nio.file.Path; +import java.util.Optional; + +public class WriteFileTask implements Task { + private final Params params; + + public static class Params { + private final Path path; + private final Producer<String> contentProducer; + + private Optional<String> user = Optional.empty(); + private Optional<String> group = Optional.empty(); + private Optional<String> permissions = Optional.empty(); + + public Params(Path path, Producer<String> contentProducer) { + this.path = path; + this.contentProducer = contentProducer; + } + + public Params withUser(String user) { + this.user = Optional.of(user); + return this; + } + + public Params withGroup(String group) { + this.group = Optional.of(group); + return this; + } + + /** + * @param permissions of the form "rwxr-x---". + */ + public Params withPermissions(String permissions) { + this.permissions = Optional.of(permissions); + return this; + } + } + + public WriteFileTask(Params params) { + this.params = params; + Task.validateVariant(variantName()); + } + + @Override + public String variantName() { + return filenameStem(params.path); + } + + static String filenameStem(Path path) { + String filename = path.getFileName().toString(); + int i = filename.indexOf('.'); + if (i >= 0) { + return filename.substring(0, i); + } else { + return filename; + } + } + + @Override + public boolean execute(TaskContext context) { + final FileSystemPath path = context.getFileSystem().withPath(params.path); + if (path.isRegularFile()) { + return false; + } + + context.executeSubtask(new MakeDirectoryTask(params.path.getParent()).withParents()); + + path.writeUtf8File(params.contentProducer.call()) + .setPermissions("rw-r--r--") + .setOwner("root") + .setGroup("root"); + + // TODO: Only return true if file changed. + return true; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/package-info.java new file mode 100644 index 00000000000..433884139f6 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.task; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index 582992869aa..2d351141103 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import org.junit.Test; @@ -48,7 +47,7 @@ public class NodeAdminImplTest { private final DockerOperations dockerOperations = mock(DockerOperations.class); private final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); + private final Runnable aclMaintainer = mock(Runnable.class); private final ManualClock clock = new ManualClock(); private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, |