diff options
author | Valerij Fredriksen <valerijf@vespa.ai> | 2023-11-04 09:44:19 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerijf@vespa.ai> | 2023-11-04 09:44:44 +0100 |
commit | 808831f0b51fd4f85239e5659cc1fb8200c70994 (patch) | |
tree | 0017c10f1adcf750442fcd98a513fa52910155da | |
parent | df7854e4c340ea814ca5cdf128fc31828a4b543b (diff) |
Reapply "Move node-admin"
347 files changed, 0 insertions, 29805 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 33cf9914a07..7902f9a017e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,7 +153,6 @@ add_subdirectory(metrics) add_subdirectory(metrics-proxy) add_subdirectory(model-evaluation) add_subdirectory(model-integration) -add_subdirectory(node-admin) add_subdirectory(node-repository) add_subdirectory(orchestrator) add_subdirectory(persistence) diff --git a/dist/vespa.spec b/dist/vespa.spec index 03f789b5852..78b89f7ebdc 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -513,14 +513,12 @@ fi %exclude %{_prefix}/libexec/vespa/common-env.sh %exclude %{_prefix}/libexec/vespa/vespa-wrapper %exclude %{_prefix}/libexec/vespa/find-pid -%exclude %{_prefix}/libexec/vespa/node-admin.sh %exclude %{_prefix}/libexec/vespa/standalone-container.sh %exclude %{_prefix}/libexec/vespa/vespa-curl-wrapper %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/access %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/configserver -%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/node-admin %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/search %{_prefix}/man %{_prefix}/sbin @@ -634,17 +632,6 @@ fi %dir %{_prefix}/lib/jars %{_prefix}/lib/jars/config-model-fat.jar -%files node-admin -%if %{_defattr_is_vespa_vespa} -%defattr(-,%{_vespa_user},%{_vespa_group},-) -%endif -%dir %{_prefix} -%dir %{_prefix}/conf -%{_prefix}/conf/node-admin-app -%dir %{_prefix}/libexec -%dir %{_prefix}/libexec/vespa -%{_prefix}/libexec/vespa/node-admin.sh - %files jars %if %{_defattr_is_vespa_vespa} %defattr(-,%{_vespa_user},%{_vespa_group},-) diff --git a/node-admin/.gitignore b/node-admin/.gitignore deleted file mode 100644 index adbb97d2d31..00000000000 --- a/node-admin/.gitignore +++ /dev/null @@ -1 +0,0 @@ -data/
\ No newline at end of file diff --git a/node-admin/CMakeLists.txt b/node-admin/CMakeLists.txt deleted file mode 100644 index 1056a09976a..00000000000 --- a/node-admin/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install(DIRECTORY DESTINATION logs/vespa/node-admin) -install(FILES target/node-admin-jar-with-dependencies.jar DESTINATION conf/node-admin-app/components) -install_symlink(lib/jars/application-model-jar-with-dependencies.jar conf/node-admin-app/components/application-model-jar-with-dependencies.jar) -install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/node-admin-app/components/flags-jar-with-dependencies.jar) -install(FILES src/main/application/services.xml DESTINATION conf/node-admin-app) -install(PROGRAMS src/main/sh/node-admin.sh DESTINATION libexec/vespa) diff --git a/node-admin/OWNERS b/node-admin/OWNERS deleted file mode 100644 index e131dacde49..00000000000 --- a/node-admin/OWNERS +++ /dev/null @@ -1 +0,0 @@ -hakonhall diff --git a/node-admin/README.md b/node-admin/README.md deleted file mode 100644 index d366400ea7e..00000000000 --- a/node-admin/README.md +++ /dev/null @@ -1,4 +0,0 @@ -<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Node Admin - -Manages docker containers that run different applications on a host. diff --git a/node-admin/pom.xml b/node-admin/pom.xml deleted file mode 100644 index 75adc2fb380..00000000000 --- a/node-admin/pom.xml +++ /dev/null @@ -1,175 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>8-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - - <artifactId>node-admin</artifactId> - <version>8-SNAPSHOT</version> - <packaging>container-plugin</packaging> - <name>${project.artifactId}</name> - - <dependencies> - <!-- Provided --> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-dev</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-athenz</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>flags</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-apache-http-client-bundle</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - - <!-- Compile --> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>orchestrator-restapi</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.velocity</groupId> - <artifactId>velocity-engine-core</artifactId> - <exclusions> - <exclusion> - <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. --> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>http-utils</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - - <!-- Test --> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>application</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>application-model</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>orchestrator</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>node-repository</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>service-monitor</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <!-- Needed for node repo mock --> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zkfacade</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-test</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <attachBundleArtifact>true</attachBundleArtifact> - </configuration> - </plugin> - <plugin> - <!-- Explicit for IntelliJ to detect correct language level from parent --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml deleted file mode 100644 index d1ea4f2b2ff..00000000000 --- a/node-admin/src/main/application/services.xml +++ /dev/null @@ -1,12 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0" xmlns:preprocess="properties"> - <container id="node-admin" version="1.0"> - <!-- Please update container test when changing this file --> - <accesslog type="json" fileNamePattern="logs/vespa/node-admin/access-json.log.%Y%m%d%H%M%S" symlinkName="access-json.log" compressOnRotation="true" compressionType="zstd" bufferSize='262144' queueSize='1024'/> - - <component id="metrics" class="com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics" bundle="node-admin"/> - - <preprocess:include file="variant.xml" required="false"/> - </container> -</services> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java deleted file mode 100644 index 034c7a381ed..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.container.ContainerId; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.logging.Logger; - -/** - * Represents a cgroup in the control group v2 hierarchy, see - * <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">Control Group v2</a>. - * - * @author hakonhall - */ -public class Cgroup { - private static final Logger logger = Logger.getLogger(Cgroup.class.getName()); - - private static final Map<String, Consumer<UnixPath>> cgroupDirectoryCallbacks = new HashMap<>(); - - private final Path root; - private final Path relativePath; - - public static Cgroup root(FileSystem fileSystem) { - return new Cgroup(fileSystem.getPath("/sys/fs/cgroup"), fileSystem.getPath("")); - } - - private Cgroup(Path root, Path relativePath) { - this.root = root.normalize(); - this.relativePath = this.root.relativize(this.root.resolve(relativePath).normalize()); - if (this.relativePath.toString().equals("..") || this.relativePath.toString().startsWith("../")) { - throw new IllegalArgumentException("Invalid cgroup relative path: " + relativePath); - } - } - - /** Whether this cgroup actually exists in the kernel / on the file system. */ - public boolean exists() { return unixPath().resolve("cgroup.controllers").exists(); } - - /** Creates this cgroup if it does not already exist, and return this. */ - public Cgroup create() { - if (unixPath().createDirectory()) { - // cgroup automatically creates various files in a newly created cgroup directory. A unit test may simulate - // this by registering consumers before the test is run. - Consumer<UnixPath> callback = cgroupDirectoryCallbacks.get(relativePath.toString()); - if (callback != null) - callback.accept(unixPath()); - } - return this; - } - - /** Whether v2 cgroup is enabled on this host. */ - public boolean v2CgroupIsEnabled() { return resolveRoot().exists(); } - - /** - * Resolve the given path against the path of this cgroup, and return the resulting cgroup. - * If the given path is absolute, it is resolved against the root of the cgroup hierarchy. - */ - public Cgroup resolve(String path) { - Path effectivePath = fileSystem().getPath(path); - if (effectivePath.isAbsolute()) { - return new Cgroup(root, fileSystem().getPath("/").relativize(effectivePath)); - } else { - return new Cgroup(root, relativePath.resolve(path)); - } - } - - /** Returns the root cgroup, possibly this. */ - public Cgroup resolveRoot() { return isRoot() ? this : new Cgroup(root, fileSystem().getPath("")); } - - /** Returns the cgroup of a system service assuming this is the root, e.g. vespa-host-admin -> system.slice/vespa-host-admin.service. */ - public Cgroup resolveSystemService(String name) { return resolve("system.slice").resolve(serviceNameOf(name)); } - - /** Returns the root cgroup of the given Podman container. */ - public Cgroup resolveContainer(ContainerId containerId) { return resolve("/machine.slice/libpod-" + containerId + ".scope/container"); } - - /** Returns the root cgroup of the container, or otherwise the root cgroup. */ - public Cgroup resolveRoot(Optional<ContainerId> containerId) { return containerId.map(this::resolveContainer).orElseGet(this::resolveRoot); } - - /** Returns the absolute path to this cgroup. */ - public Path path() { return root.resolve(relativePath); } - - /** Returns the UnixPath of {@link #path()}. */ - public UnixPath unixPath() { return new UnixPath(path()); } - - public String read(String filename) { - return unixPath().resolve(filename).readUtf8File(); - } - - public Optional<String> readIfExists(String filename) { - return unixPath().resolve(filename).readUtf8FileIfExists().map(String::strip); - } - - public List<String> readLines(String filename) { - return unixPath().resolve(filename).readUtf8File().lines().toList(); - } - - public Optional<Integer> readIntIfExists(String filename) { - return unixPath().resolve(filename).readUtf8FileIfExists().map(String::strip).map(Integer::parseInt); - } - - public Size readSize(String filename) { return Size.from(read(filename).stripTrailing()); } - - public boolean convergeFileContent(TaskContext context, String filename, String content, boolean apply) { - UnixPath path = unixPath().resolve(filename); - String currentContent = path.readUtf8File(); - if (ensureSuffixNewline(currentContent).equals(ensureSuffixNewline(content))) return false; - - if (apply) { - context.recordSystemModification(logger, "Updating " + path + " from '" + currentContent.stripTrailing() + - "' to '" + content.stripTrailing() + "'"); - path.writeUtf8File(content); - } - return true; - } - - /** The kernel appears to append a newline if none exist, when writing to files in cgroupfs. */ - private static String ensureSuffixNewline(String content) { - return content.endsWith("\n") ? content : content + "\n"; - } - - /** Returns an instance representing core interface files (cgroup.* files). */ - public CgroupCore core() { return new CgroupCore(this); } - - /** Returns the CPU controller of this cgroup (cpu.* files). */ - public CpuController cpu() { return new CpuController(this); } - - /** Returns the memory controller of this cgroup (memory.* files). */ - public MemoryController memory() { return new MemoryController(this); } - - /** Returns the IO controller of this cgroup (io.* files). */ - public IoController io() { return new IoController(this); } - - /** - * Wraps {@code command} to ensure it is executed in this cgroup. - * - * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p> - */ - public String[] wrapCommandForExecutionInCgroup(String... command) { - String[] fullCommand = new String[3 + command.length]; - fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec"; - fullCommand[1] = "-g"; - fullCommand[2] = relativePath.toString(); - System.arraycopy(command, 0, fullCommand, 3, command.length); - return fullCommand; - } - - public static void unitTesting_atCgroupCreation(String relativePath, Consumer<UnixPath> callback) { - cgroupDirectoryCallbacks.put(relativePath, callback); - } - - private boolean isRoot() { return relativePath.toString().isEmpty(); } - - private static String serviceNameOf(String name) { - return name.indexOf('.') == -1 ? name + ".service" : name; - } - - private FileSystem fileSystem() { return root.getFileSystem(); } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java deleted file mode 100644 index ecee819cc66..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import java.util.List; - -/** - * Utility methods for accessing the cgroup core interface files, i.e. all cgroup.* files. - * - * @author hakonhall - */ -public class CgroupCore { - private final Cgroup cgroup; - - CgroupCore(Cgroup cgroup) { this.cgroup = cgroup; } - - public List<Integer> getPidsInCgroup() { - return cgroup.readLines("cgroup.procs") - .stream() - .map(Integer::parseInt) - .toList(); - } - - /** Whether the given PID is a member of this cgroup. */ - public boolean isMember(int pid) { - return getPidsInCgroup().contains(pid); - } - - /** Move the given PID to this cgroup, but return false if it was already a member. */ - public boolean addMember(int pid) { - if (isMember(pid)) return false; - cgroup.unixPath().resolve("cgroup.procs").writeUtf8File(Integer.toString(pid)); - return true; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java deleted file mode 100644 index 5ca8a84cad6..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static java.lang.Integer.parseInt; - -/** - * Represents a cgroup v2 CPU controller, i.e. all cpu.* files. - * - * @author hakonhall - */ -public class CpuController { - private final Cgroup cgroup; - - CpuController(Cgroup cgroup) { - this.cgroup = cgroup; - } - - /** - * The maximum bandwidth limit of the format "QUOTA PERIOD", which indicates that the cgroup may consume - * up to QUOTA in each PERIOD duration. A quota of "max" indicates no limit. - */ - public record Max(Size quota, int period) { - public String toFileContent() { return quota + " " + period + '\n'; } - } - - /** - * Returns the maximum CPU usage, or empty if cgroup is not found. - * - * @see Max - */ - public Optional<Max> readMax() { - return cgroup.readIfExists("cpu.max") - .map(content -> { - String[] parts = content.strip().split(" "); - return new Max(Size.from(parts[0]), parseInt(parts[1])); - }); - } - - /** - * Update CPU quota and period for the given container ID. Set quota to -1 value for unlimited. - * - * @see #readMax() - * @see Max - */ - public boolean updateMax(TaskContext context, int quota, int period) { - Max max = new Max(quota < 0 ? Size.max() : Size.from(quota), period); - return cgroup.convergeFileContent(context, "cpu.max", max.toFileContent(), true); - } - - /** @return The weight in the range [1, 10000], or empty if not found. */ - private Optional<Integer> readWeight() { - return cgroup.readIntIfExists("cpu.weight"); - } - - /** @return The number of shares allocated to this cgroup for purposes of CPU time scheduling, or empty if not found. */ - public Optional<Integer> readShares() { - return readWeight().map(CpuController::weightToShares); - } - - public boolean updateShares(TaskContext context, int shares) { - return cgroup.convergeFileContent(context, "cpu.weight", sharesToWeight(shares) + "\n", true); - } - - // Must be same as in crun: https://github.com/containers/crun/blob/72c6e60ade0e4716fe2d8353f0d97d72cc8d1510/src/libcrun/cgroup.c#L3061 - // TODO: Migrate to weights - public static int sharesToWeight(int shares) { return (int) (1 + ((shares - 2L) * 9999) / 262142); } - public static int weightToShares(int weight) { return (int) (2 + ((weight - 1L) * 262142) / 9999); } - - public enum StatField { - TOTAL_USAGE_USEC("usage_usec"), - USER_USAGE_USEC("user_usec"), - SYSTEM_USAGE_USEC("system_usec"), - TOTAL_PERIODS("nr_periods"), - THROTTLED_PERIODS("nr_throttled"), - THROTTLED_TIME_USEC("throttled_usec"); - - private final String name; - - StatField(String name) { - this.name = name; - } - - long parseValue(String value) { - return Long.parseLong(value); - } - - static Optional<StatField> fromField(String fieldName) { - return Arrays.stream(values()) - .filter(field -> fieldName.equals(field.name)) - .findFirst(); - } - } - - public Map<StatField, Long> readStats() { - return cgroup.readLines("cpu.stat") - .stream() - .map(line -> line.split("\\s+")) - .filter(parts -> parts.length == 2) - .flatMap(parts -> StatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) - .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java deleted file mode 100644 index f6676347605..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import ai.vespa.validation.Validation; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static java.lang.Integer.parseInt; - -/** - * Represents a cgroup v2 IO controller, i.e. all io.* files. - * - * @author freva - */ -public class IoController { - private static final Logger logger = Logger.getLogger(IoController.class.getName()); - private final Cgroup cgroup; - - IoController(Cgroup cgroup) { - this.cgroup = cgroup; - } - - public record Device(int major, int minor) implements Comparable<Device> { - public Device { - // https://www.halolinux.us/kernel-architecture/representation-of-major-and-minor-numbers.html - Validation.requireInRange(major, "device major", 0, 0xFFF); - Validation.requireInRange(minor, "device minor", 0, 0xFFFFF); - } - - private String toFileContent() { return major + ":" + minor; } - private static Device fromString(String device) { - String[] parts = device.split(":"); - return new Device(parseInt(parts[0]), parseInt(parts[1])); - } - - @Override - public int compareTo(Device o) { - return major != o.major ? Integer.compare(major, o.major) : Integer.compare(minor, o.minor); - } - } - - /** - * Defines max allowed IO: - * <ul> - * <li><b>rbps</b>: Read bytes per seconds</li> - * <li><b>riops</b>: Read IO operations per seconds</li> - * <li><b>wbps</b>: Write bytes per seconds</li> - * <li><b>wiops</b>: Write IO operations per seconds</li> - * </ul>. - */ - public record Max(Size rbps, Size wbps, Size riops, Size wiops) { - public static Max UNLIMITED = new Max(Size.max(), Size.max(), Size.max(), Size.max()); - - // Keys can be specified in any order, this is the order they are outputted in from io.max - // https://github.com/torvalds/linux/blob/c1a515d3c0270628df8ae5f5118ba859b85464a2/block/blk-throttle.c#L1541 - private String toFileContent() { return "rbps=%s wbps=%s riops=%s wiops=%s".formatted(rbps, wbps, riops, wiops); } - - public static Max fromString(String max) { - String[] parts = max.split(" "); - Size rbps = Size.max(), riops = Size.max(), wbps = Size.max(), wiops = Size.max(); - for (String part : parts) { - if (part.isEmpty()) continue; - String[] kv = part.split("="); - if (kv.length != 2) throw new IllegalArgumentException("Invalid io.max format: " + max); - switch (kv[0]) { - case "rbps" -> rbps = Size.from(kv[1]); - case "riops" -> riops = Size.from(kv[1]); - case "wbps" -> wbps = Size.from(kv[1]); - case "wiops" -> wiops = Size.from(kv[1]); - default -> throw new IllegalArgumentException("Unknown key " + kv[0]); - } - } - return new Max(rbps, wbps, riops, wiops); - } - } - - /** - * Returns the maximum allowed IO usage, by device, or empty if cgroup is not found. - * - * @see Max - */ - public Optional<Map<Device, Max>> readMax() { - return cgroup.readIfExists("io.max") - .map(content -> content - .lines() - .map(line -> { - String[] parts = line.strip().split(" ", 2); - return Map.entry(Device.fromString(parts[0]), Max.fromString(parts[1])); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - } - - public boolean updateMax(TaskContext context, Device device, Max max) { - Max prevMax = readMax() - .map(maxByDevice -> maxByDevice.get(device)) - .orElse(Max.UNLIMITED); - if (prevMax.equals(max)) return false; - - UnixPath path = cgroup.unixPath().resolve("io.max"); - context.recordSystemModification(logger, "Updating %s for device %s from '%s' to '%s'", - path, device.toFileContent(), prevMax.toFileContent(), max.toFileContent()); - path.writeUtf8File(device.toFileContent() + ' ' + max.toFileContent() + '\n'); - return true; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java deleted file mode 100644 index 28da683ea69..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import java.util.List; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -/** - * Represents a cgroup v2 memory controller, i.e. all memory.* files. - * - * @author hakonhall - */ -public class MemoryController { - private final Cgroup cgroup; - - MemoryController(Cgroup cgroup) { - this.cgroup = cgroup; - } - - /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ - public Size readMax() { - return cgroup.readSize("memory.max"); - } - - /** @return The total amount of memory currently being used by the cgroup and its descendants, in bytes. */ - public Size readCurrent() { - return cgroup.readSize("memory.current"); - } - - /** @return The total amount of memory currently being used by the cgroup and its descendants, in bytes. */ - public Optional<Size> readCurrentIfExists() { - return cgroup.readIfExists("memory.current").map(Size::from); - } - - public Stats readStat() { - var lines = cgroup.readLines("memory.stat"); - return new Stats( - Size.from(readField(lines, "file")), Size.from(readField(lines, "sock")), Size.from(readField(lines, "slab")), - Size.from(readField(lines, "slab_reclaimable")), Size.from(readField(lines, "anon"))); - } - - public Optional<Pressure> readPressureIfExists() { - return cgroup.readIfExists("memory.pressure") - .map(fileContent -> - new Pressure( - readPressureField(fileContent, "some"), - readPressureField(fileContent, "full") - ) - ); - } - - private static String readField(List<String> lines, String fieldName) { - return lines.stream() - .map(line -> line.split("\\s+")) - .filter(fields -> fields.length == 2) - .filter(fields -> fieldName.equals(fields[0])) - .map(fields -> fields[1]) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("No such field: " + fieldName)); - } - - /** - * Fetches the avg60 value from the specified type, i.e. "some" or "full". - */ - private static Double readPressureField(String fileContent, String type) { - var pattern = Pattern.compile(type + ".*avg60=(?<avg60>\\d+\\.\\d+).*"); - return Stream.of(fileContent.split("\n")) - .map(pattern::matcher) - .filter(Matcher::matches) - .map(matcher -> matcher.group("avg60")) - .findFirst() - .map(Double::parseDouble) - .orElseThrow(() -> new IllegalArgumentException("No such field: " + type)); - } - - /** - * @param file Number of bytes used to cache filesystem data, including tmpfs and shared memory. - * @param sock Amount of memory used in network transmission buffers. - * @param slab Amount of memory used for storing in-kernel data structures. - * @param slabReclaimable Part of "slab" that might be reclaimed, such as dentries and inodes. - * @param anon Amount of memory used in anonymous mappings such as brk(), sbrk(), and mmap(MAP_ANONYMOUS). - */ - public record Stats(Size file, Size sock, Size slab, Size slabReclaimable, Size anon) {} - - /** - * @param some The avg60 value of the "some" pressure level. - * @param full The avg60 value of the "full" pressure level. - */ - public record Pressure(double some, double full) {} -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java deleted file mode 100644 index d89db56e4d2..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import java.util.Objects; - -/** - * Represents a number of bytes or possibly "max". - * - * @author hakonhall - */ -public class Size { - private static final String MAX = "max"; - private static final Size MAX_SIZE = new Size(true, 0); - - private final boolean max; - private final long value; - - public static Size max() { - return MAX_SIZE; - } - - public static Size from(long value) { - return new Size(false, value); - } - - public static Size from(String value) { - return value.equals(MAX) ? MAX_SIZE : new Size(false, Long.parseLong(value)); - } - - private Size(boolean max, long value) { - this.max = max; - this.value = value; - } - - public boolean isMax() { - return max; - } - - /** Returns the value, i.e. the number of "bytes" if applicable. Throws if this is max. */ - public long value() { - if (max) throw new IllegalStateException("Value is max"); - return value; - } - - public String toFileContent() { return toString() + '\n'; } - - @Override - public String toString() { return max ? MAX : Long.toString(value); } - - public boolean isGreaterThan(Size that) { - if (that.max) return false; - if (this.max) return true; - return this.value > that.value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Size size = (Size) o; - return max == size.max && value == size.value; - } - - @Override - public int hashCode() { - return Objects.hash(max, value); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java deleted file mode 100644 index b4c1a5228f8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -/** - * @author hakonhall - */ -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java deleted file mode 100644 index 64c6b19b8bb..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.vespa.athenz.api.AthenzIdentity; - -import java.net.URI; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Information necessary to e.g. establish communication with the config servers - * - * @author hakon - */ -public class ConfigServerInfo { - private final URI loadBalancerEndpoint; - private final AthenzIdentity configServerIdentity; - private final Function<String, URI> configServerHostnameToUriMapper; - private final List<URI> configServerURIs; - - public ConfigServerInfo(URI loadBalancerEndpoint, List<String> configServerHostNames, - AthenzIdentity configServerAthenzIdentity) { - this.loadBalancerEndpoint = loadBalancerEndpoint; - this.configServerIdentity = configServerAthenzIdentity; - this.configServerHostnameToUriMapper = hostname -> URI.create("https://" + hostname + ":4443"); - this.configServerURIs = configServerHostNames.stream() - .map(configServerHostnameToUriMapper) - .toList(); - } - - public List<URI> getConfigServerUris() { - return configServerURIs; - } - - public URI getConfigServerUri(String hostname) { - return configServerHostnameToUriMapper.apply(hostname); - } - - public URI getLoadBalancerEndpoint() { - return loadBalancerEndpoint; - } - - public AthenzIdentity getConfigServerIdentity() { - return configServerIdentity; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java deleted file mode 100644 index 492020b7ae4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -/** - * <p>This class is thread unsafe: All method calls MUST be exclusive and serialized.</p> - * - * <dl> - * <dt>In a specialized environment it is possible to provide a richer context than TaskContext:</dt> - * <dd>- Define a subclass T of TaskContext with the additional functionality.</dd> - * <dd>- Define task classes that implement IdempotentTask<T>.</dd> - * </dl> - */ -public interface IdempotentTask<T extends TaskContext> { - /** - * <p>A short id of the task to e.g. identify the task in the log.</p> - * - * <p>Prefer PascalCase and without white-space.</p> - * - * <p>Example: "EnableDocker"</p> - */ - default String name() { return getClass().getSimpleName(); } - - /** - * <p>Execute an administrative task to converge towards some ideal state, whether it is - * system state or in-memory Java state.</p> - * - * <p>converge() must be idempotent: it may be called any number of times, or - * interrupted at any time e.g. by `kill -9`.</p> - * - * <p>converge() is not thread safe: The caller must ensure there is at most one invocation - * of converge() at any given time.</p> - * - * @return false if already converged, i.e. was a no-op. A typical sequence of converge() - * calls on a IdempotentTask will consist of: - * - Any number of calls that throws an exception due to some issues. Assuming - * no exceptions were thrown, or the issue eventually resolved itself... - * (convergence failure) - * - Returns true once (converged just now) - * - Returns false for all further calls (already converged) - * @throws RuntimeException (or a subclass) if the task is unable to converge. - */ - boolean converge(T context); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java deleted file mode 100644 index 0e8fdb6e1f6..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import java.util.logging.Level; -import java.util.logging.Logger; - -public interface TaskContext { - /** - * Record a system modification. IdempotentTask is supposed to converge the system (files, - * directory permission, iptable rules, etc) to some wanted state. It is especially important - * to produce a truthful log of system changes to understand what may or may not be going on. - * - * All tasks should: - * 1. Record any and all modifications to the system - * 2. Avoid recording system interactions that does not actually change the system. - * 3. Record system modifications as early as possible and preferably before they are - * performed (sometimes this is not possible). - * - * @param logger Used to log the modification to help locate the source of the modification. - * @param message Description of the modification, e.g. "Changing owner of /foo from alice - * to bob". - */ - void recordSystemModification(Logger logger, String message); - default void recordSystemModification(Logger logger, String messageFormat, Object... args) { - recordSystemModification(logger, String.format(messageFormat, args)); - } - - /** - * Log message at Level.INFO, scoped to denote the current task. The message may - * also be directed to status pages or similar. - * - * Please do not call this too many times as that spams the log. Typically a task may call - * this zero times, or up to a few times. - * - * Do not log a message that is also recorded with recordSystemModification. - */ - default void log(Logger logger, String message) { - log(logger, Level.INFO, message); - } - - default void log(Logger logger, String messageFormat, Object... args) { - log(logger, String.format(messageFormat, args)); - } - - void log(Logger logger, Level level, String message); - - void log(Logger logger, Level level, String message, Throwable throwable); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java deleted file mode 100644 index beedb56941a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.component; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class TestTaskContext implements TaskContext { - private final List<String> systemModifications = new ArrayList<>(); - - @Override - public void recordSystemModification(Logger logger, String description) { - systemModifications.add(description); - } - - @Override - public void log(Logger logger, Level level, String message) { - logger.log(level, message); - } - - @Override - public void log(Logger logger, Level level, String message, Throwable throwable) { - logger.log(level, message, throwable); - } - - public List<String> getSystemModificationLog() { - return systemModifications; - } - - public void clearSystemModificationLog() { - systemModifications.clear(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java deleted file mode 100644 index 53cb32300b4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java deleted file mode 100644 index b401e2f3d08..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import java.net.URI; -import java.time.Duration; -import java.util.Optional; - -/** - * Interface to execute basic HTTP/HTTPS request against config server(s) - * - * @author freva - */ -public interface ConfigServerApi extends AutoCloseable { - - /** - * The result of sending a request to a config server results in a jackson response or exception. If a response - * is returned, an instance of this interface is conferred to discard the result and try the next config server, - * unless it was the last attempt. - * - * @param <T> the type of the returned jackson response - */ - interface RetryPolicy<T> { - boolean tryNextConfigServer(URI configServerEndpoint, T response); - } - - class Params<T> { - private Optional<Duration> connectionTimeout = Optional.empty(); - - private RetryPolicy<T> retryPolicy = (configServerEndpoint, response) -> false; - - public Params() {} - - /** Set the socket connect and read timeouts. */ - public Params<T> setConnectionTimeout(Duration connectionTimeout) { - this.connectionTimeout = Optional.of(connectionTimeout); - return this; - } - - public Optional<Duration> getConnectionTimeout() { return connectionTimeout; } - - /** Set the retry policy to use against the config servers. */ - public Params<T> setRetryPolicy(RetryPolicy<T> retryPolicy) { - this.retryPolicy = retryPolicy; - return this; - } - - public RetryPolicy<T> getRetryPolicy() { return retryPolicy; } - } - - <T> T get(String path, Class<T> wantedReturnType, Params<T> params); - default <T> T get(String path, Class<T> wantedReturnType) { - return get(path, wantedReturnType, new Params<>()); - } - - <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params); - default <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { - return post(path, bodyJsonPojo, wantedReturnType, new Params<>()); - } - - <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType, Params<T> params); - default <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType) { - return put(path, bodyJsonPojo, wantedReturnType, new Params<>()); - } - - <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params); - default <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { - return patch(path, bodyJsonPojo, wantedReturnType, new Params<>()); - } - - <T> T delete(String path, Class<T> wantedReturnType, Params<T> params); - default <T> T delete(String path, Class<T> wantedReturnType) { - return delete(path, wantedReturnType, new Params<>()); - } - - /** Close the underlying HTTP client and any threads this class might have started. */ - @Override - void close(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java deleted file mode 100644 index b645e993a05..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import ai.vespa.util.http.hc4.SslConnectionSocketFactory; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory; -import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; -import com.yahoo.yolean.Exceptions; -import org.apache.http.HttpHeaders; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPatch; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.HostnameVerifier; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * Retries request on config server a few times before giving up. Assumes that all requests should be sent with - * content-type application/json - * - * @author dybdahl - * @author bjorncs - */ -public class ConfigServerApiImpl implements ConfigServerApi { - - private static final Logger logger = Logger.getLogger(ConfigServerApiImpl.class.getName()); - private static final RequestConfig DEFAULT_REQUEST_CONFIG = RequestConfig.custom() - .setConnectionRequestTimeout(1_000) // connection from connection manager - .setConnectTimeout(10_000) // establishment of connection - .setSocketTimeout(10_000) // waiting for data - .build(); - - private final ObjectMapper mapper = new ObjectMapper(); - - private final List<URI> configServers; - - private final CloseableHttpClient client; - - public static ConfigServerApiImpl create(ConfigServerInfo info, - ServiceIdentityProvider provider, - HostnameVerifier hostnameVerifier) { - return new ConfigServerApiImpl( - info.getConfigServerUris(), - hostnameVerifier, - provider); - } - - public static ConfigServerApiImpl createFor(URI uri, - ServiceIdentityProvider provider, - HostnameVerifier hostnameVerifier) { - return new ConfigServerApiImpl(List.of(uri), hostnameVerifier, provider); - } - - private ConfigServerApiImpl(Collection<URI> configServers, - HostnameVerifier verifier, - ServiceIdentityProvider identityProvider) { - this(configServers, createClient(SslConnectionSocketFactory.of(new ServiceIdentitySslSocketFactory(identityProvider), verifier))); - } - - private ConfigServerApiImpl(Collection<URI> configServers, CloseableHttpClient client) { - this.configServers = randomizeConfigServerUris(configServers); - this.client = client; - } - - public static ConfigServerApiImpl createForTesting(List<URI> configServerHosts) { - return new ConfigServerApiImpl(configServerHosts, createClient(SslConnectionSocketFactory.of())); - } - - static ConfigServerApiImpl createForTestingWithClient(List<URI> configServerHosts, - CloseableHttpClient client) { - return new ConfigServerApiImpl(configServerHosts, client); - } - - interface CreateRequest { - HttpUriRequest createRequest(URI configServerUri) throws JsonProcessingException, UnsupportedEncodingException; - } - - private <T> T tryAllConfigServers(CreateRequest requestFactory, Class<T> wantedReturnType, Params<T> params) { - T lastResult = null; - Exception lastException = null; - - for (URI configServer : configServers) { - var request = Exceptions.uncheck(() -> requestFactory.createRequest(configServer)); - try (CloseableHttpResponse response = client.execute(request)) { - var responseBody = EntityUtils.toString(response.getEntity()); - HttpException.handleStatusCode(response.getStatusLine().getStatusCode(), - request.getMethod() + " " + request.getURI() + - " failed with response '" + responseBody + "'"); - - T result; - try { - result = mapper.readValue(responseBody, wantedReturnType); - } catch (IOException e) { - throw new UncheckedIOException("Failed parse response from config server", e); - } - - if (params.getRetryPolicy().tryNextConfigServer(configServer, result)) { - lastResult = result; - lastException = null; - } else { - return result; - } - } catch (HttpException e) { - if (!e.isRetryable()) throw e; - lastResult = null; - lastException = e; - } catch (Exception e) { - lastResult = null; - lastException = e; - if (configServers.size() == 1) break; - - // Failure to communicate with a config server is not abnormal during upgrades - if (ConnectionException.isKnownConnectionException(e)) { - logger.info("Failed to connect to " + configServer + ", will try next: " + e.getMessage()); - } else { - logger.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); - } - } - } - - if (lastResult != null) { - logger.warning("Giving up after trying all config servers: returning result: " + lastResult); - return lastResult; - } - - String prefix = configServers.size() == 1 ? - "Request against " + configServers.get(0) + " failed: " : - "All requests against the config servers (" + configServers + ") failed, last as follows: "; - throw ConnectionException.handleException(prefix, lastException); - } - - @Override - public <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) { - Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params); - return tryAllConfigServers(configServer -> { - HttpPut put = new HttpPut(configServer.resolve(path)); - requestConfigOverride.ifPresent(put::setConfig); - setContentTypeToApplicationJson(put); - if (bodyJsonPojo.isPresent()) { - put.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo.get()))); - } - return put; - }, wantedReturnType, params); - } - - @Override - public <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) { - Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params); - return tryAllConfigServers(configServer -> { - HttpPatch patch = new HttpPatch(configServer.resolve(path)); - requestConfigOverride.ifPresent(patch::setConfig); - setContentTypeToApplicationJson(patch); - patch.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo))); - return patch; - }, wantedReturnType, params); - } - - @Override - public <T> T delete(String path, Class<T> wantedReturnType, Params<T> params) { - Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params); - return tryAllConfigServers(configServer -> { - HttpDelete delete = new HttpDelete(configServer.resolve(path)); - requestConfigOverride.ifPresent(delete::setConfig); - return delete; - }, wantedReturnType, params); - } - - @Override - public <T> T get(String path, Class<T> wantedReturnType, Params<T> params) { - Optional<RequestConfig> requestConfig = getRequestConfigOverride(params); - return tryAllConfigServers(configServer -> { - HttpGet get = new HttpGet(configServer.resolve(path)); - requestConfig.ifPresent(get::setConfig); - return get; - }, wantedReturnType, params); - } - - @Override - public <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) { - Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params); - return tryAllConfigServers(configServer -> { - HttpPost post = new HttpPost(configServer.resolve(path)); - requestConfigOverride.ifPresent(post::setConfig); - setContentTypeToApplicationJson(post); - post.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo))); - return post; - }, wantedReturnType, params); - } - - @Override - public void close() { - // Need to do try and catch, using e.g. uncheck(client::close) might fail because - // components are deconstructed in random order and if the bundle containing uncheck has been - // unloaded it will fail with NoClassDefFoundError - try { - client.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private void setContentTypeToApplicationJson(HttpRequestBase request) { - request.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); - } - - private static CloseableHttpClient createClient(SSLConnectionSocketFactory socketFactory) { - Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create() - .register("http", PlainConnectionSocketFactory.getSocketFactory()) - .register("https", socketFactory) - .build(); - - PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - cm.setMaxTotal(200); // Increase max total connections to 200, which should be enough - - // Have experienced hang in socket read, which may have been because of - // system defaults, therefore set explicit timeouts. - return HttpClientBuilder.create() - .setDefaultRequestConfig(DEFAULT_REQUEST_CONFIG) - .disableAutomaticRetries() - .disableConnectionState() // Share connections between subsequent requests. - .setUserAgent("node-admin") // Node-repository depends on this value to identify agent of node-admin/host-admin requests - .setConnectionManager(cm) - .build(); - } - - private static <T> Optional<RequestConfig> getRequestConfigOverride(Params<T> params) { - if (params.getConnectionTimeout().isEmpty()) return Optional.empty(); - - RequestConfig.Builder builder = RequestConfig.copy(DEFAULT_REQUEST_CONFIG); - - params.getConnectionTimeout().ifPresent(connectionTimeout -> { - builder.setConnectTimeout((int) connectionTimeout.toMillis()); - builder.setSocketTimeout((int) connectionTimeout.toMillis()); - }); - - return Optional.of(builder.build()); - } - - // Shuffle config server URIs to balance load - private static List<URI> randomizeConfigServerUris(Collection<URI> configServerUris) { - List<URI> shuffledConfigServerHosts = new ArrayList<>(configServerUris); - Collections.shuffle(shuffledConfigServerHosts); - return shuffledConfigServerHosts; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java deleted file mode 100644 index 8c6212f83f4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.yahoo.vespa.flags.FlagRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.configserver.state.State; - -/** - * The available (and implemented) APIs of the config server - * - * @author freva - */ -public interface ConfigServerClients { - /** Get handle to /nodes/v2/ REST API */ - NodeRepository nodeRepository(); - - /** Get handle to /orchestrator/v1/ REST API */ - Orchestrator orchestrator(); - - /** Get handle to the /state/v1 REST API */ - State state(); - - /** Get handle to the /flags/v1 REST API */ - FlagRepository flagRepository(); - - /** Get handle to the /cores/v1 REST API */ - Cores cores(); - - void stop(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerException.java deleted file mode 100644 index e957a56c0ae..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerException.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -/** - * @author hakonhall - */ -public class ConfigServerException extends RuntimeException { - public ConfigServerException(String message) { super(message); } - public ConfigServerException(String message, Throwable cause) { super(message, cause); } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java deleted file mode 100644 index 86c52efe282..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConnectionException.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import org.apache.http.NoHttpResponseException; - -import java.io.EOFException; -import java.net.SocketException; -import java.net.SocketTimeoutException; - -/** - * @author freva - */ -@SuppressWarnings("serial") -public class ConnectionException extends ConvergenceException { - - private ConnectionException(String message, Throwable cause) { - super(message, cause, true); - } - - /** - * Returns {@link ConnectionException} if the given Throwable is of a known and well understood error or - * a RuntimeException with the given exception as cause otherwise. - */ - public static RuntimeException handleException(String prefix, Throwable t) { - if (isKnownConnectionException(t)) - return new ConnectionException(prefix + t.getMessage(), t); - - return new RuntimeException(prefix, t); - } - - static boolean isKnownConnectionException(Throwable t) { - for (; t != null; t = t.getCause()) { - if (t instanceof SocketException || - t instanceof SocketTimeoutException || - t instanceof NoHttpResponseException || - t instanceof EOFException) - return true; - } - - return false; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java deleted file mode 100644 index 64b1ebe239d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; - -import javax.ws.rs.core.Response; - -/** - * @author hakonhall - */ -@SuppressWarnings("serial") -public class HttpException extends ConvergenceException { - - private final boolean isRetryable; - - private HttpException(int statusCode, String message, boolean isRetryable) { - super("HTTP status code " + statusCode + ": " + message, null, !isRetryable); - this.isRetryable = isRetryable; - } - - private HttpException(Response.Status status, String message, boolean isRetryable) { - super(status.toString() + " (" + status.getStatusCode() + "): " + message, null, !isRetryable); - this.isRetryable = isRetryable; - } - - boolean isRetryable() { - return isRetryable; - } - - /** - * Returns on success. - * @throws HttpException for all non-expected status codes. - */ - static void handleStatusCode(int statusCode, String message) { - Response.Status status = Response.Status.fromStatusCode(statusCode); - if (status == null) { - throw new HttpException(statusCode, message, true); - } - - switch (status.getFamily()) { - case SUCCESSFUL: return; - case CLIENT_ERROR: - switch (status) { - case FORBIDDEN: - throw new ForbiddenException(message); - case NOT_FOUND: - throw new NotFoundException(message); - case CONFLICT: - // A response body is assumed to be present, and - // will later be interpreted as an error. - return; - } - throw new HttpException(status, message, false); - } - - // Other errors like server-side errors are assumed to be NOT retryable, - // in case retries would put additional load on a bogged down server. - throw new HttpException(status, message, false); - } - - public static class NotFoundException extends HttpException { - public NotFoundException(String message) { - super(Response.Status.NOT_FOUND, message, false); - } - } - - public static class ForbiddenException extends HttpException { - public ForbiddenException(String message) { - super(Response.Status.FORBIDDEN, message, false); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java deleted file mode 100644 index 8ee346246ae..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.yahoo.vespa.flags.FlagRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoresImpl; -import com.yahoo.vespa.hosted.node.admin.configserver.flags.RealFlagRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorImpl; -import com.yahoo.vespa.hosted.node.admin.configserver.state.State; -import com.yahoo.vespa.hosted.node.admin.configserver.state.StateImpl; - -/** - * {@link ConfigServerClients} using the default implementation for the various clients, - * and backed by a {@link ConfigServerApi}. - * - * @author freva - */ -public class RealConfigServerClients implements ConfigServerClients { - private final ConfigServerApi configServerApi; - private final NodeRepository nodeRepository; - private final Orchestrator orchestrator; - private final State state; - private final RealFlagRepository flagRepository; - private final Cores cores; - - /** - * @param configServerApi the backend API to use - will be closed at {@link #stop()}. - */ - public RealConfigServerClients(ConfigServerApi configServerApi) { - this.configServerApi = configServerApi; - nodeRepository = new RealNodeRepository(configServerApi); - orchestrator = new OrchestratorImpl(configServerApi); - state = new StateImpl(configServerApi); - flagRepository = new RealFlagRepository(configServerApi); - cores = new CoresImpl(configServerApi); - } - - @Override - public NodeRepository nodeRepository() { - return nodeRepository; - } - - @Override - public Orchestrator orchestrator() { - return orchestrator; - } - - @Override - public State state() { - return state; - } - - @Override - public FlagRepository flagRepository() { - return flagRepository; - } - - @Override - public Cores cores() { - return cores; - } - - @Override - public void stop() { - configServerApi.close(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/StandardConfigServerResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/StandardConfigServerResponse.java deleted file mode 100644 index c967091ccbf..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/StandardConfigServerResponse.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Strings; - -/** - * @author hakonhall - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class StandardConfigServerResponse { - @JsonProperty("message") public String message; - @JsonProperty("error-code") public String errorCode; - - public void throwOnError(String detail) { - if (!Strings.isNullOrEmpty(errorCode)) - throw new ConfigServerException(detail + ": " + message + " " + errorCode); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java deleted file mode 100644 index 2f4595ce5d1..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.cores; - -import com.yahoo.config.provision.DockerImage; - -import java.nio.file.Path; -import java.time.Instant; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * @author hakonhall - */ -public class CoreDumpMetadata { - public enum Type { CORE_DUMP, JVM_HEAP, OOM } - - private Type type; - private String binPath; - private Instant created; - private List<String> backtrace; - private List<String> backtraceAllThreads; - private Path coreDumpPath; - private String decryptionToken; - private String kernelVersion; - private String cpuMicrocodeVersion; - private DockerImage dockerImage; - private String vespaVersion; - - public CoreDumpMetadata() {} - - public Optional<Type> type() { return Optional.ofNullable(type); } - public Optional<String> binPath() { return Optional.ofNullable(binPath); } - public Optional<Instant> created() { return Optional.ofNullable(created); } - public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); } - public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); } - public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); } - public Optional<String> decryptionToken() { return Optional.ofNullable(decryptionToken); } - public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); } - public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); } - public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); } - public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); } - - public CoreDumpMetadata setType(Type type) { this.type = type; return this; } - public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; } - public CoreDumpMetadata setCreated(Instant created) { this.created = created; return this; } - public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; } - public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; } - public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; } - public CoreDumpMetadata setDecryptionToken(String decryptionToken) { this.decryptionToken = decryptionToken; return this; } - public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; } - public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; } - public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; } - public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; } - - @Override - public String toString() { - return "CoreDumpMetadata{" + - "type=" + type + - ", binPath=" + binPath + - ", created=" + created + - ", backtrace=" + backtrace + - ", backtraceAllThreads=" + backtraceAllThreads + - ", coreDumpPath=" + coreDumpPath + - ", decryptionToken=" + decryptionToken + - ", kernelVersion='" + kernelVersion + '\'' + - ", cpuMicrocodeVersion='" + cpuMicrocodeVersion + '\'' + - ", dockerImage=" + dockerImage + - ", vespaVersion=" + vespaVersion + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CoreDumpMetadata metadata = (CoreDumpMetadata) o; - return type == metadata.type && - Objects.equals(binPath, metadata.binPath) && - Objects.equals(created, metadata.created) && - Objects.equals(backtrace, metadata.backtrace) && - Objects.equals(backtraceAllThreads, metadata.backtraceAllThreads) && - Objects.equals(coreDumpPath, metadata.coreDumpPath) && - Objects.equals(decryptionToken, metadata.decryptionToken) && - Objects.equals(kernelVersion, metadata.kernelVersion) && - Objects.equals(cpuMicrocodeVersion, metadata.cpuMicrocodeVersion) && - Objects.equals(dockerImage, metadata.dockerImage) && - Objects.equals(vespaVersion, metadata.vespaVersion); - } - - @Override - public int hashCode() { - return Objects.hash(type, binPath, created, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion, - cpuMicrocodeVersion, dockerImage, vespaVersion); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/Cores.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/Cores.java deleted file mode 100644 index b168c6f6dbe..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/Cores.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.cores; - -import com.yahoo.config.provision.HostName; - -/** - * @author hakonhall - */ -public interface Cores { - /** - * @param hostname Hostname of the node that produced the core. - * @param id The ID (aka UUID aka docid) of the core. - * @param metadata Core dump metadata. - */ - void report(HostName hostname, String id, CoreDumpMetadata metadata); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java deleted file mode 100644 index 200fe97283c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresImpl.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.cores; - -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest; - -/** - * @author hakonhall - */ -public class CoresImpl implements Cores { - private final ConfigServerApi configServerApi; - - public CoresImpl(ConfigServerApi configServerApi) { - this.configServerApi = configServerApi; - } - - @Override - public void report(HostName hostname, String id, CoreDumpMetadata metadata) { - var request = new ReportCoreDumpRequest().fillFrom(metadata); - String uriPath = "/cores/v1/report/" + hostname.value() + "/" + id; - configServerApi.post(uriPath, request, StandardConfigServerResponse.class) - .throwOnError("Failed to report core dump at " + metadata.coredumpPath()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java deleted file mode 100644 index 435367cd1ca..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.time.Instant; -import java.util.List; -import java.util.Optional; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Jackson class of JSON request, with names of fields verified in unit test. - * - * @author hakonhall - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ReportCoreDumpRequest { - private static final ObjectMapper objectMapper = new ObjectMapper(); - - public List<String> backtrace; - public List<String> backtrace_all_threads; - public Long created; - public String type; - public String bin_path; - public String coredump_path; - public String cpu_microcode_version; - public String decryption_token; - public String docker_image; - public String kernel_version; - public String vespa_version; - - public ReportCoreDumpRequest() {} - - /** Fill this from metadata and return this. */ - @JsonIgnore - public ReportCoreDumpRequest fillFrom(CoreDumpMetadata metadata) { - metadata.type().ifPresent(type -> this.type = type.name()); - metadata.binPath().ifPresent(binPath -> this.bin_path = binPath); - metadata.created().ifPresent(created -> this.created = created.toEpochMilli()); - metadata.backtrace().ifPresent(backtrace -> this.backtrace = List.copyOf(backtrace)); - metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> this.backtrace_all_threads = List.copyOf(backtraceAllThreads)); - metadata.coredumpPath().ifPresent(coredumpPath -> this.coredump_path = coredumpPath.toString()); - metadata.decryptionToken().ifPresent(decryptionToken -> this.decryption_token = decryptionToken); - metadata.kernelVersion().ifPresent(kernelVersion -> this.kernel_version = kernelVersion); - metadata.cpuMicrocodeVersion().ifPresent(cpuMicrocodeVersion -> this.cpu_microcode_version = cpuMicrocodeVersion); - metadata.dockerImage().ifPresent(dockerImage -> this.docker_image = dockerImage.asString()); - metadata.vespaVersion().ifPresent(vespaVersion -> this.vespa_version = vespaVersion); - return this; - } - - @JsonIgnore - public void populateMetadata(CoreDumpMetadata metadata, FileSystem fileSystem) { - if (type != null) metadata.setType(CoreDumpMetadata.Type.valueOf(type)); - if (bin_path != null) metadata.setBinPath(bin_path); - if (created != null) metadata.setCreated(Instant.ofEpochMilli(created)); - if (backtrace != null) metadata.setBacktrace(backtrace); - if (backtrace_all_threads != null) metadata.setBacktraceAllThreads(backtrace_all_threads); - if (coredump_path != null) metadata.setCoreDumpPath(fileSystem.getPath(coredump_path)); - if (decryption_token != null) metadata.setDecryptionToken(decryption_token); - if (kernel_version != null) metadata.setKernelVersion(kernel_version); - if (cpu_microcode_version != null) metadata.setCpuMicrocodeVersion(cpu_microcode_version); - if (docker_image != null) metadata.setDockerImage(DockerImage.fromString(docker_image)); - if (vespa_version != null) metadata.setVespaVersion(vespa_version); - } - - @JsonIgnore - public void save(Path path) { - String serialized = uncheck(() -> objectMapper.writeValueAsString(this)); - uncheck(() -> Files.writeString(path, serialized)); - } - - @JsonIgnore - public static Optional<ReportCoreDumpRequest> load(Path path) { - final String serialized; - try { - serialized = Files.readString(path); - } catch (NoSuchFileException e) { - return Optional.empty(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return Optional.of(uncheck(() -> objectMapper.readValue(serialized, ReportCoreDumpRequest.class))); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/package-info.java deleted file mode 100644 index d8a07b2b0df..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -/** - * @author hakonhall - */ -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.configserver.cores; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java deleted file mode 100644 index 97c93e6a48a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.flags; - -import com.yahoo.vespa.flags.FlagId; -import com.yahoo.vespa.flags.FlagRepository; -import com.yahoo.vespa.flags.json.FlagData; -import com.yahoo.vespa.flags.json.wire.WireFlagDataList; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; - -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * @author hakonhall - */ -public class RealFlagRepository implements FlagRepository { - private final ConfigServerApi configServerApi; - - public RealFlagRepository(ConfigServerApi configServerApi) { - this.configServerApi = configServerApi; - } - - @Override - public Map<FlagId, FlagData> getAllFlagData() { - WireFlagDataList list = configServerApi.get("/flags/v1/data?recursive=true", WireFlagDataList.class); - return FlagData.listFromWire(list).stream().collect(Collectors.toMap(FlagData::id, Function.identity())); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java deleted file mode 100644 index b5f1bc2a3bc..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.configserver.flags; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java deleted file mode 100644 index dd13658ba27..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.google.common.net.InetAddresses; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; - -import java.net.InetAddress; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * This class represents an ACL for a specific container instance. - * - * @author mpolden - * @author smorgrav - */ -public class Acl { - - public static final Acl EMPTY = new Acl(Set.of(), Set.of(), Set.of(), Set.of()); - - private final Set<Node> trustedNodes; - private final Set<Integer> trustedPorts; - private final Set<Integer> trustedUdpPorts; - private final Set<String> trustedNetworks; - - /** - * @param trustedPorts TCP Ports to trust - * @param trustedUdpPorts UDP ports to trust - * @param trustedNodes Nodes to trust - * @param trustedNetworks Networks (in CIDR notation) to trust - */ - public Acl(Set<Integer> trustedPorts, Set<Integer> trustedUdpPorts, Set<Node> trustedNodes, Set<String> trustedNetworks) { - this.trustedNodes = copyOfNullable(trustedNodes); - this.trustedPorts = copyOfNullable(trustedPorts); - this.trustedUdpPorts = copyOfNullable(trustedUdpPorts); - this.trustedNetworks = copyOfNullable(trustedNetworks); - } - - public Acl(Set<Integer> trustedPorts, Set<Node> trustedNodes) { - this(trustedPorts, Set.of(), trustedNodes, Set.of()); - } - - public List<String> toRules(IPVersion ipVersion) { - List<String> rules = new LinkedList<>(); - - // We reject with rules instead of using policies - rules.add("-P INPUT ACCEPT"); - rules.add("-P FORWARD ACCEPT"); - rules.add("-P OUTPUT ACCEPT"); - - // Allow packets belonging to established connections - rules.add( "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT"); - - // Allow any loopback traffic - rules.add("-A INPUT -i lo -j ACCEPT"); - - // Allow ICMP packets. See http://shouldiblockicmp.com/ - rules.add("-A INPUT -p " + ipVersion.icmpProtocol() + " -j ACCEPT"); - - // Allow trusted ports if any - if (!trustedPorts.isEmpty()) { - rules.add("-A INPUT -p tcp -m multiport --dports " + joinPorts(trustedPorts) + " -j ACCEPT"); - } - - // Allow trusted UDP ports if any - if (!trustedUdpPorts.isEmpty()) { - rules.add("-A INPUT -p udp -m multiport --dports " + joinPorts(trustedUdpPorts) + " -j ACCEPT"); - } - - // Allow traffic from trusted nodes, limited to specific ports, if any - getTrustedNodes(ipVersion).stream() - .map(node -> { - StringBuilder rule = new StringBuilder(); - rule.append("-A INPUT -s ") - .append(node.inetAddressString()) - .append(ipVersion.singleHostCidr()); - if (!node.ports.isEmpty()) { - rule.append(" -p tcp -m multiport --dports ") - .append(joinPorts(node.ports())); - } - rule.append(" -j ACCEPT"); - return rule.toString(); - }) - .sorted() - .forEach(rules::add); - - // Allow traffic from trusted networks - addressesOf(ipVersion, trustedNetworks).stream() - .map(network -> "-A INPUT -s " + network + " -j ACCEPT") - .sorted() - .forEach(rules::add); - - // We reject instead of dropping to give us an easier time to figure out potential network issues - rules.add("-A INPUT -j REJECT --reject-with " + ipVersion.icmpPortUnreachable()); - - return Collections.unmodifiableList(rules); - } - - private static String joinPorts(Collection<Integer> ports) { - return ports.stream().sorted().map(String::valueOf).collect(Collectors.joining(",")); - } - - public Set<Node> getTrustedNodes() { - return trustedNodes; - } - - public Set<Node> getTrustedNodes(IPVersion ipVersion) { - return trustedNodes.stream() - .filter(node -> ipVersion.match(node.inetAddress())) - .collect(Collectors.toSet()); - } - - public Set<Integer> getTrustedPorts() { - return trustedPorts; - } - - public Set<Integer> getTrustedUdpPorts() { - return trustedUdpPorts; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Acl acl = (Acl) o; - return trustedNodes.equals(acl.trustedNodes) && - trustedPorts.equals(acl.trustedPorts) && - trustedUdpPorts.equals(acl.trustedUdpPorts) && - trustedNetworks.equals(acl.trustedNetworks); - } - - @Override - public int hashCode() { - return Objects.hash(trustedNodes, trustedPorts, trustedUdpPorts, trustedNetworks); - } - - @Override - public String toString() { - return "Acl{" + - "trustedNodes=" + trustedNodes + - ", trustedPorts=" + trustedPorts + - ", trustedUdpPorts=" + trustedUdpPorts + - ", trustedNetworks=" + trustedNetworks + - '}'; - } - - private static Set<String> addressesOf(IPVersion version, Set<String> addresses) { - return addresses.stream() - .filter(version::match) - .collect(Collectors.toUnmodifiableSet()); - } - - private static <T> Set<T> copyOfNullable(Set<T> set) { - return Optional.ofNullable(set).map(Set::copyOf).orElseGet(Set::of); - } - - public record Node(String hostname, InetAddress inetAddress, Set<Integer> ports) { - - public Node(String hostname, String ipAddress, Set<Integer> ports) { - this(hostname, InetAddresses.forString(ipAddress), ports); - } - - public String inetAddressString() { - return InetAddresses.toAddrString(inetAddress); - } - - @Override - public String toString() { - return "Node{" + - "hostname='" + hostname + '\'' + - ", inetAddress=" + inetAddress + - ", ports=" + ports + - '}'; - } - } - - public static class Builder { - - private final Set<Node> trustedNodes = new HashSet<>(); - private final Set<Integer> trustedPorts = new HashSet<>(); - private final Set<Integer> trustedUdpPorts = new HashSet<>(); - private final Set<String> trustedNetworks = new HashSet<>(); - - public Builder() { } - - public Builder(Acl acl) { - trustedNodes.addAll(acl.trustedNodes); - trustedPorts.addAll(acl.trustedPorts); - trustedNetworks.addAll(acl.trustedNetworks); - } - - public Builder withTrustedNode(Node node) { - trustedNodes.add(node); - return this; - } - - public Builder withTrustedNode(String hostname, String ipAddress) { - return withTrustedNode(hostname, ipAddress, Set.of()); - } - - public Builder withTrustedNode(String hostname, String ipAddress, Set<Integer> ports) { - return withTrustedNode(new Node(hostname, ipAddress, ports)); - } - - public Builder withTrustedNode(String hostname, InetAddress inetAddress, Set<Integer> ports) { - return withTrustedNode(new Node(hostname, inetAddress, ports)); - } - - public Builder withTrustedPorts(Integer... ports) { - trustedPorts.addAll(List.of(ports)); - return this; - } - - public Builder withTrustedUdpPorts(Integer... ports) { - trustedUdpPorts.addAll(List.of(ports)); - return this; - } - - public Builder withTrustedNetworks(Set<String> networks) { - trustedNetworks.addAll(networks); - return this; - } - - public Acl build() { - return new Acl(trustedPorts, trustedUdpPorts, trustedNodes, trustedNetworks); - } - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java deleted file mode 100644 index 47b59414efd..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.host.FlavorOverrides; - -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * @author freva - */ -public class AddNode { - - public final String hostname; - public final String id; - public final Optional<String> parentHostname; - public final Optional<String> nodeFlavor; - public final Optional<FlavorOverrides> flavorOverrides; - public final Optional<NodeResources> nodeResources; - public final NodeType nodeType; - public final Set<String> ipAddresses; - public final Set<String> additionalIpAddresses; - - public static AddNode forHost(String hostname, String id, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { - return new AddNode(hostname, id, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses); - } - - public static AddNode forNode(String hostname, String id, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { - return new AddNode(hostname, id, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); - } - - private AddNode(String hostname, String id, Optional<String> parentHostname, - Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides, - Optional<NodeResources> nodeResources, - NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { - this.hostname = hostname; - this.id = id; - this.parentHostname = parentHostname; - this.nodeFlavor = nodeFlavor; - this.flavorOverrides = flavorOverrides; - this.nodeResources = nodeResources; - this.nodeType = nodeType; - this.ipAddresses = ipAddresses; - this.additionalIpAddresses = additionalIpAddresses; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AddNode addNode = (AddNode) o; - return Objects.equals(hostname, addNode.hostname) && - Objects.equals(id, addNode.id) && - Objects.equals(parentHostname, addNode.parentHostname) && - Objects.equals(nodeFlavor, addNode.nodeFlavor) && - Objects.equals(flavorOverrides, addNode.flavorOverrides) && - Objects.equals(nodeResources, addNode.nodeResources) && - nodeType == addNode.nodeType && - Objects.equals(ipAddresses, addNode.ipAddresses) && - Objects.equals(additionalIpAddresses, addNode.additionalIpAddresses); - } - - @Override - public int hashCode() { - return Objects.hash(hostname, id, parentHostname, nodeFlavor, flavorOverrides, nodeResources, nodeType, ipAddresses, additionalIpAddresses); - } - - @Override - public String toString() { - return "AddNode{" + - "hostname='" + hostname + '\'' + - ", id=" + id + - ", parentHostname=" + parentHostname + - ", nodeFlavor='" + nodeFlavor + '\'' + - ", flavorOverrides='" + flavorOverrides + '\'' + - ", nodeResources='" + nodeResources + '\'' + - ", nodeType=" + nodeType + - ", ipAddresses=" + ipAddresses + - ", additionalIpAddresses=" + additionalIpAddresses + - '}'; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Event.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Event.java deleted file mode 100644 index 554e9f4df13..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Event.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import java.time.Instant; -import java.util.Objects; - -/** - * @author freva - */ -public class Event { - private final String agent; - private final String type; - private final Instant at; - - public Event(String agent, String type, Instant at) { - this.agent = Objects.requireNonNull(agent); - this.type = Objects.requireNonNull(type); - this.at = Objects.requireNonNull(at); - } - - public String agent() { - return agent; - } - - public String type() { - return type; - } - - public Instant at() { - return at; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Event event1 = (Event) o; - return agent.equals(event1.agent) && type.equals(event1.type) && at.equals(event1.at); - } - - @Override - public int hashCode() { - return Objects.hash(agent, type, at); - } - - @Override - public String toString() { - return "Event{" + - "agent='" + agent + '\'' + - ", type='" + type + '\'' + - ", at=" + at + - '}'; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NoSuchNodeException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NoSuchNodeException.java deleted file mode 100644 index 4c77019f9ba..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NoSuchNodeException.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -public class NoSuchNodeException extends NodeRepositoryException { - public NoSuchNodeException(String message) { - super(message); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java deleted file mode 100644 index 9b22de3f279..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.fasterxml.jackson.databind.JsonNode; -import com.yahoo.component.Version; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport; - -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.TreeMap; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * A node in the node repository is modified by setting which attributes to modify in this class, - * and then patching the node repository node through {@link NodeRepository#updateNodeAttributes(String, NodeAttributes)}. - * - * @author Haakon Dybdahl - * @author Valerij Fredriksen - */ -public class NodeAttributes { - - private Optional<String> hostId = Optional.empty(); - private Optional<Long> restartGeneration = Optional.empty(); - private Optional<Long> rebootGeneration = Optional.empty(); - private Optional<DockerImage> dockerImage = Optional.empty(); - private Optional<Version> vespaVersion = Optional.empty(); - private Optional<Version> currentOsVersion = Optional.empty(); - private Optional<Instant> currentFirmwareCheck = Optional.empty(); - private List<TrustStoreItem> trustStore = List.of(); - private Optional<WireguardKey> wireguardPubkey = Optional.empty(); - /** The list of reports to patch. A null value is used to remove the report. */ - private Map<String, JsonNode> reports = new TreeMap<>(); - - public NodeAttributes() { } - - public NodeAttributes withHostId(String hostId) { - this.hostId = Optional.of(hostId); - return this; - } - - public NodeAttributes withRestartGeneration(Optional<Long> restartGeneration) { - this.restartGeneration = restartGeneration; - return this; - } - - public NodeAttributes withRestartGeneration(long restartGeneration) { - return withRestartGeneration(Optional.of(restartGeneration)); - } - - public NodeAttributes withRebootGeneration(long rebootGeneration) { - this.rebootGeneration = Optional.of(rebootGeneration); - return this; - } - - public NodeAttributes withDockerImage(DockerImage dockerImage) { - this.dockerImage = Optional.of(dockerImage); - return this; - } - - public NodeAttributes withVespaVersion(Version vespaVersion) { - this.vespaVersion = Optional.of(vespaVersion); - return this; - } - - public NodeAttributes withCurrentOsVersion(Version currentOsVersion) { - this.currentOsVersion = Optional.of(currentOsVersion); - return this; - } - - public NodeAttributes withCurrentFirmwareCheck(Instant currentFirmwareCheck) { - this.currentFirmwareCheck = Optional.of(currentFirmwareCheck); - return this; - } - - public NodeAttributes withTrustStore(List<TrustStoreItem> trustStore) { - this.trustStore = List.copyOf(trustStore); - return this; - } - - public NodeAttributes withWireguardPubkey(WireguardKey wireguardPubkey) { - this.wireguardPubkey = Optional.of(wireguardPubkey); - return this; - } - - public NodeAttributes withReports(Map<String, JsonNode> nodeReports) { - this.reports = new TreeMap<>(nodeReports); - return this; - } - - public NodeAttributes withReport(String reportId, JsonNode jsonNode) { - reports.put(reportId, jsonNode); - return this; - } - - public NodeAttributes withReportRemoved(String reportId) { - reports.put(reportId, null); - return this; - } - - public Optional<String> getHostId() { - return hostId; - } - - public Optional<Long> getRestartGeneration() { - return restartGeneration; - } - - public Optional<Long> getRebootGeneration() { - return rebootGeneration; - } - - public Optional<DockerImage> getDockerImage() { - return dockerImage; - } - - public Optional<Version> getVespaVersion() { - return vespaVersion; - } - - public Optional<Version> getCurrentOsVersion() { - return currentOsVersion; - } - - public Optional<Instant> getCurrentFirmwareCheck() { - return currentFirmwareCheck; - } - - public List<TrustStoreItem> getTrustStore() { - return trustStore; - } - - public Optional<WireguardKey> getWireguardPubkey() { return wireguardPubkey; } - - public Map<String, JsonNode> getReports() { - return reports; - } - - public <T extends BaseReport> Optional<T> getReport(String reportId, Class<T> classInstance) { - return Optional.ofNullable(reports.get(reportId)).map(jn -> BaseReport.fromJsonNode(jn, classInstance)); - } - - @Override - public int hashCode() { - return Objects.hash(hostId, restartGeneration, rebootGeneration, dockerImage, vespaVersion, currentOsVersion, - currentFirmwareCheck, trustStore, wireguardPubkey, reports); - } - - public boolean isEmpty() { - return equals(new NodeAttributes()); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof NodeAttributes other)) { - return false; - } - - return Objects.equals(hostId, other.hostId) - && Objects.equals(restartGeneration, other.restartGeneration) - && Objects.equals(rebootGeneration, other.rebootGeneration) - && Objects.equals(dockerImage, other.dockerImage) - && Objects.equals(vespaVersion, other.vespaVersion) - && Objects.equals(currentOsVersion, other.currentOsVersion) - && Objects.equals(currentFirmwareCheck, other.currentFirmwareCheck) - && Objects.equals(trustStore, other.trustStore) - && Objects.equals(wireguardPubkey, other.wireguardPubkey) - && Objects.equals(reports, other.reports); - } - - @Override - public String toString() { - return Stream.of(hostId.map(id -> "hostId=" + id), - restartGeneration.map(gen -> "restartGeneration=" + gen), - rebootGeneration.map(gen -> "rebootGeneration=" + gen), - dockerImage.map(img -> "dockerImage=" + img.asString()), - vespaVersion.map(ver -> "vespaVersion=" + ver.toFullString()), - currentOsVersion.map(ver -> "currentOsVersion=" + ver.toFullString()), - currentFirmwareCheck.map(at -> "currentFirmwareCheck=" + at), - Optional.ofNullable(trustStore.isEmpty() ? null : "trustStore=" + trustStore), - Optional.ofNullable(wireguardPubkey.isEmpty() ? null : "wireguardPubkey=" + wireguardPubkey), - Optional.ofNullable(reports.isEmpty() ? null : "reports=" + reports)) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.joining(", ", "{", "}")); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java deleted file mode 100644 index c70eccfa0ea..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import java.util.Objects; - -/** - * @author freva - */ -public class NodeMembership { - private final ClusterType clusterType; - private final String clusterId; - private final String group; - private final int index; - private final boolean retired; - - public NodeMembership(String clusterType, String clusterId, String group, int index, boolean retired) { - this.clusterType = new ClusterType(clusterType); - this.clusterId = clusterId; - this.group = group; - this.index = index; - this.retired = retired; - } - - public ClusterType type() { - return clusterType; - } - - public String clusterId() { - return clusterId; - } - - public String group() { - return group; - } - - public int index() { return index; } - - public boolean isRetired() { - return retired; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - NodeMembership that = (NodeMembership) o; - - if (index != that.index) return false; - if (retired != that.retired) return false; - if (!clusterType.equals(that.clusterType)) return false; - if (!clusterId.equals(that.clusterId)) return false; - return group.equals(that.group); - - } - - @Override - public int hashCode() { - int result = clusterType.hashCode(); - result = 31 * result + clusterId.hashCode(); - result = 31 * result + group.hashCode(); - result = 31 * result + index; - result = 31 * result + (retired ? 1 : 0); - return result; - } - - @Override - public String toString() { - return "Membership {" + - " clusterType = " + clusterType + - " clusterId = " + clusterId + - " group = " + group + - " index = " + index + - " retired = " + retired + - " }"; - } - - public static class ClusterType { - private final String type; - - private ClusterType(String type) { - this.type = Objects.requireNonNull(type); - } - - public boolean isAdmin() { return "admin".equals(type); } - public boolean isContent() { return "content".equals(type); } - public boolean isCombined() { return "combined".equals(type); } - public boolean isContainer() { return "container".equals(type); } - public boolean hasContainer() { return isContainer() || isCombined(); } - public boolean hasContent() { return isContent() || isCombined(); } - - public String value() { - return type; - } - - @Override - public String toString() { - return type; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ClusterType that = (ClusterType) o; - return type.equals(that.type); - } - - @Override - public int hashCode() { - return type.hashCode(); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java deleted file mode 100644 index c45c2dd9578..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport; - -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeMap; -import java.util.stream.Collectors; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * API of node reports within node-admin. - * - * @author hakonhall - */ -public class NodeReports { - private static final ObjectMapper mapper = new ObjectMapper(); - - private final Map<String, JsonNode> reports = new TreeMap<>(); - - public NodeReports() { } - - public NodeReports(NodeReports reports) { - this.reports.putAll(reports.reports); - } - - private NodeReports(Map<String, JsonNode> reports) { - this.reports.putAll(Objects.requireNonNull(reports)); - } - - public static NodeReports fromMap(Map<String, JsonNode> reports) { - return new NodeReports(reports); - } - - public void setReport(String reportId, JsonNode jsonNode) { - reports.put(reportId, jsonNode); - } - - public boolean hasReport(String reportId) { return reports.containsKey(reportId); } - - public <T> Optional<T> getReport(String reportId, Class<T> jacksonClass) { - return Optional.ofNullable(reports.get(reportId)).map(r -> uncheck(() -> mapper.treeToValue(r, jacksonClass))); - } - - /** Gets all reports of the given types and deserialize with the given jacksonClass. */ - public <T> TreeMap<String, T> getReports(Class<T> jacksonClass, BaseReport.Type... types) { - Set<BaseReport.Type> typeSet = Set.of(types); - - return reports.entrySet().stream() - .filter(entry -> { - JsonNode reportType = entry.getValue().findValue(BaseReport.TYPE_FIELD); - if (reportType == null || !reportType.isTextual()) return false; - Optional<BaseReport.Type> type = BaseReport.Type.deserialize(reportType.asText()); - return type.map(typeSet::contains).orElse(false); - }) - .collect(Collectors.toMap( - entry -> entry.getKey(), - entry -> uncheck(() -> mapper.treeToValue(entry.getValue(), jacksonClass)), - (x,y) -> x, // resolves key collisions - cannot happen. - TreeMap::new - )); - } - - public void removeReport(String reportId) { - if (reports.containsKey(reportId)) { - reports.put(reportId, null); - } - } - - public Map<String, JsonNode> getRawMap() { - return new TreeMap<>(reports); - } - - /** Apply the override to this. null value means removing report. */ - public void updateFromRawMap(Map<String, JsonNode> override) { - override.forEach((reportId, jsonNode) -> { - if (jsonNode == null) { - reports.remove(reportId); - } else { - reports.put(reportId, jsonNode); - } - }); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NodeReports that = (NodeReports) o; - return Objects.equals(reports, that.reports); - } - - @Override - public int hashCode() { - return Objects.hash(reports); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java deleted file mode 100644 index ac1f8ec059f..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; - -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * @author stiankri - */ -public interface NodeRepository { - - void addNodes(List<AddNode> nodes); - - List<NodeSpec> getNodes(String baseHostName); - - default NodeSpec getNode(String hostName) { - return getOptionalNode(hostName).orElseThrow(() -> new NoSuchNodeException(hostName + " not found in node-repo")); - } - - Optional<NodeSpec> getOptionalNode(String hostName); - - Map<String, Acl> getAcls(String hostname); - - List<WireguardPeer> getExclavePeers(); - - List<WireguardPeer> getConfigserverPeers(); - - void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes); - - void setNodeState(String hostName, NodeState nodeState); - - default void reboot(String hostname) { - throw new UnsupportedOperationException("Rebooting not supported in " + getClass().getName()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java deleted file mode 100644 index f46f0c9f446..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryException.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; - -public class NodeRepositoryException extends ConvergenceException { - public NodeRepositoryException(String message) { - super(message, null, true); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java deleted file mode 100644 index 3700b57d169..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ /dev/null @@ -1,880 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.fasterxml.jackson.databind.JsonNode; -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.config.provision.WireguardKeyWithTimestamp; -import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; - -import java.net.URI; -import java.time.Instant; -import java.util.EnumSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; -import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; - -/** - * @author stiankri - */ -public class NodeSpec { - - private final String hostname; - private final String id; - private final NodeState state; - private final NodeType type; - private final CloudAccount cloudAccount; - private final String flavor; - - private final Optional<DockerImage> wantedDockerImage; - private final Optional<DockerImage> currentDockerImage; - - private final Optional<Version> wantedVespaVersion; - private final Optional<Version> currentVespaVersion; - - private final Optional<Version> wantedOsVersion; - private final Optional<Version> currentOsVersion; - - private final Optional<Long> wantedRestartGeneration; - private final Optional<Long> currentRestartGeneration; - - private final long wantedRebootGeneration; - private final long currentRebootGeneration; - - private final Optional<Instant> wantedFirmwareCheck; - private final Optional<Instant> currentFirmwareCheck; - - private final Optional<String> modelName; - - private final OrchestratorStatus orchestratorStatus; - private final Optional<ApplicationId> owner; - private final Optional<NodeMembership> membership; - - private final NodeResources resources; - private final NodeResources realResources; - private final Set<String> ipAddresses; - private final Set<String> additionalIpAddresses; - - private final NodeReports reports; - private final List<Event> events; - - private final Optional<String> parentHostname; - private final Optional<URI> archiveUri; - - private final Optional<ApplicationId> exclusiveTo; - - private final List<TrustStoreItem> trustStore; - - private final Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestamp; - - private final boolean wantToRebuild; - - public NodeSpec( - String hostname, - String id, - Optional<DockerImage> wantedDockerImage, - Optional<DockerImage> currentDockerImage, - NodeState state, - NodeType type, - CloudAccount cloudAccount, - String flavor, - Optional<Version> wantedVespaVersion, - Optional<Version> currentVespaVersion, - Optional<Version> wantedOsVersion, - Optional<Version> currentOsVersion, - OrchestratorStatus orchestratorStatus, - Optional<ApplicationId> owner, - Optional<NodeMembership> membership, - Optional<Long> wantedRestartGeneration, - Optional<Long> currentRestartGeneration, - long wantedRebootGeneration, - long currentRebootGeneration, - Optional<Instant> wantedFirmwareCheck, - Optional<Instant> currentFirmwareCheck, - Optional<String> modelName, - NodeResources resources, - NodeResources realResources, - Set<String> ipAddresses, - Set<String> additionalIpAddresses, - NodeReports reports, - List<Event> events, - Optional<String> parentHostname, - Optional<URI> archiveUri, - Optional<ApplicationId> exclusiveTo, - List<TrustStoreItem> trustStore, - Optional<WireguardKeyWithTimestamp> wireguardPubkey, - boolean wantToRebuild) { - - if (state == NodeState.active) { - requireOptional(owner, "owner"); - requireOptional(membership, "membership"); - requireOptional(wantedVespaVersion, "wantedVespaVersion"); - requireOptional(wantedDockerImage, "wantedDockerImage"); - requireOptional(wantedRestartGeneration, "restartGeneration"); - requireOptional(currentRestartGeneration, "currentRestartGeneration"); - } - - this.hostname = Objects.requireNonNull(hostname); - this.id = Objects.requireNonNull(id); - this.wantedDockerImage = Objects.requireNonNull(wantedDockerImage); - this.currentDockerImage = Objects.requireNonNull(currentDockerImage); - this.state = Objects.requireNonNull(state); - this.type = Objects.requireNonNull(type); - this.cloudAccount = Objects.requireNonNull(cloudAccount); - this.flavor = Objects.requireNonNull(flavor); - this.modelName = Objects.requireNonNull(modelName); - this.wantedVespaVersion = Objects.requireNonNull(wantedVespaVersion); - this.currentVespaVersion = Objects.requireNonNull(currentVespaVersion); - this.wantedOsVersion = Objects.requireNonNull(wantedOsVersion); - this.currentOsVersion = Objects.requireNonNull(currentOsVersion); - this.orchestratorStatus = Objects.requireNonNull(orchestratorStatus); - this.owner = Objects.requireNonNull(owner); - this.membership = Objects.requireNonNull(membership); - this.wantedRestartGeneration = wantedRestartGeneration; - this.currentRestartGeneration = currentRestartGeneration; - this.wantedRebootGeneration = wantedRebootGeneration; - this.currentRebootGeneration = currentRebootGeneration; - this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck); - this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck); - this.resources = Objects.requireNonNull(resources); - this.realResources = Objects.requireNonNull(realResources); - this.ipAddresses = Set.copyOf(ipAddresses); - this.additionalIpAddresses = Set.copyOf(additionalIpAddresses); - this.reports = Objects.requireNonNull(reports); - this.events = List.copyOf(events); - this.parentHostname = Objects.requireNonNull(parentHostname); - this.archiveUri = Objects.requireNonNull(archiveUri); - this.exclusiveTo = Objects.requireNonNull(exclusiveTo); - this.trustStore = Objects.requireNonNull(trustStore); - this.wireguardKeyWithTimestamp = Objects.requireNonNull(wireguardPubkey); - this.wantToRebuild = wantToRebuild; - } - - public String hostname() { - return hostname; - } - - /** Returns unique node ID */ - public String id() { - return id; - } - - public NodeState state() { - return state; - } - - public NodeType type() { - return type; - } - - public CloudAccount cloudAccount() { - return cloudAccount; - } - - public String flavor() { - return flavor; - } - - public Optional<DockerImage> wantedDockerImage() { - return wantedDockerImage; - } - - public Optional<DockerImage> currentDockerImage() { - return currentDockerImage; - } - - public Optional<Version> wantedVespaVersion() { - return wantedVespaVersion; - } - - public Optional<Version> currentVespaVersion() { - return currentVespaVersion; - } - - public Optional<Version> currentOsVersion() { - return currentOsVersion; - } - - public Optional<Version> wantedOsVersion() { - return wantedOsVersion; - } - - public Optional<Long> wantedRestartGeneration() { - return wantedRestartGeneration; - } - - public Optional<Long> currentRestartGeneration() { - return currentRestartGeneration; - } - - public long wantedRebootGeneration() { - return wantedRebootGeneration; - } - - public long currentRebootGeneration() { - return currentRebootGeneration; - } - - public Optional<Instant> wantedFirmwareCheck() { - return wantedFirmwareCheck; - } - - public Optional<Instant> currentFirmwareCheck() { - return currentFirmwareCheck; - } - - public Optional<String> modelName() { - return modelName; - } - - public OrchestratorStatus orchestratorStatus() { - return orchestratorStatus; - } - - public Optional<ApplicationId> owner() { - return owner; - } - - public Optional<NodeMembership> membership() { - return membership; - } - - public NodeResources resources() { - return resources; - } - - public NodeResources realResources() { - return realResources; - } - - public double vcpu() { - return realResources.vcpu(); - } - - public double memoryGb() { - return realResources.memoryGb(); - } - - public DiskSize diskSize() { - return DiskSize.of(realResources.diskGb(), DiskSize.Unit.GB); - } - - public double diskGb() { - return realResources.diskGb(); - } - - public boolean isFastDisk() { - return realResources.diskSpeed() == fast; - } - - public double bandwidthGbps() { - return realResources.bandwidthGbps(); - } - - public Set<String> ipAddresses() { - return ipAddresses; - } - - public Set<String> additionalIpAddresses() { - return additionalIpAddresses; - } - - public NodeReports reports() { return reports; } - - public List<Event> events() { - return events; - } - - public Optional<String> parentHostname() { - return parentHostname; - } - - public Optional<URI> archiveUri() { - return archiveUri; - } - - public Optional<ApplicationId> exclusiveTo() { - return exclusiveTo; - } - - public List<TrustStoreItem> trustStore() { - return trustStore; - } - - public Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestamp() { return wireguardKeyWithTimestamp; } - - public boolean wantToRebuild() { - return wantToRebuild; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof NodeSpec that)) return false; - - return Objects.equals(hostname, that.hostname) && - Objects.equals(id, that.id) && - Objects.equals(wantedDockerImage, that.wantedDockerImage) && - Objects.equals(currentDockerImage, that.currentDockerImage) && - Objects.equals(state, that.state) && - Objects.equals(type, that.type) && - Objects.equals(cloudAccount, that.cloudAccount) && - Objects.equals(flavor, that.flavor) && - Objects.equals(modelName, that.modelName) && - Objects.equals(wantedVespaVersion, that.wantedVespaVersion) && - Objects.equals(currentVespaVersion, that.currentVespaVersion) && - Objects.equals(wantedOsVersion, that.wantedOsVersion) && - Objects.equals(currentOsVersion, that.currentOsVersion) && - Objects.equals(orchestratorStatus, that.orchestratorStatus) && - Objects.equals(owner, that.owner) && - Objects.equals(membership, that.membership) && - Objects.equals(wantedRestartGeneration, that.wantedRestartGeneration) && - Objects.equals(currentRestartGeneration, that.currentRestartGeneration) && - Objects.equals(wantedRebootGeneration, that.wantedRebootGeneration) && - Objects.equals(currentRebootGeneration, that.currentRebootGeneration) && - Objects.equals(wantedFirmwareCheck, that.wantedFirmwareCheck) && - Objects.equals(currentFirmwareCheck, that.currentFirmwareCheck) && - Objects.equals(resources, that.resources) && - Objects.equals(realResources, that.realResources) && - Objects.equals(ipAddresses, that.ipAddresses) && - Objects.equals(additionalIpAddresses, that.additionalIpAddresses) && - Objects.equals(reports, that.reports) && - Objects.equals(events, that.events) && - Objects.equals(parentHostname, that.parentHostname) && - Objects.equals(archiveUri, that.archiveUri) && - Objects.equals(exclusiveTo, that.exclusiveTo) && - Objects.equals(trustStore, that.trustStore) && - Objects.equals(wireguardKeyWithTimestamp, that.wireguardKeyWithTimestamp) && - Objects.equals(wantToRebuild, that.wantToRebuild); - } - - @Override - public int hashCode() { - return Objects.hash( - hostname, - id, - wantedDockerImage, - currentDockerImage, - state, - type, - cloudAccount, - flavor, - modelName, - wantedVespaVersion, - currentVespaVersion, - wantedOsVersion, - currentOsVersion, - orchestratorStatus, - owner, - membership, - wantedRestartGeneration, - currentRestartGeneration, - wantedRebootGeneration, - currentRebootGeneration, - wantedFirmwareCheck, - currentFirmwareCheck, - resources, - realResources, - ipAddresses, - additionalIpAddresses, - reports, - events, - parentHostname, - archiveUri, - exclusiveTo, - trustStore, - wireguardKeyWithTimestamp, - wantToRebuild); - } - - @Override - public String toString() { - return getClass().getSimpleName() + " {" - + " hostname=" + hostname - + " id=" + id - + " wantedDockerImage=" + wantedDockerImage - + " currentDockerImage=" + currentDockerImage - + " state=" + state - + " type=" + type - + " cloudAccount=" + cloudAccount - + " flavor=" + flavor - + " modelName=" + modelName - + " wantedVespaVersion=" + wantedVespaVersion - + " currentVespaVersion=" + currentVespaVersion - + " wantedOsVersion=" + wantedOsVersion - + " currentOsVersion=" + currentOsVersion - + " orchestratorStatus=" + orchestratorStatus - + " owner=" + owner - + " membership=" + membership - + " wantedRestartGeneration=" + wantedRestartGeneration - + " currentRestartGeneration=" + currentRestartGeneration - + " wantedRebootGeneration=" + wantedRebootGeneration - + " currentRebootGeneration=" + currentRebootGeneration - + " wantedFirmwareCheck=" + wantedFirmwareCheck - + " currentFirmwareCheck=" + currentFirmwareCheck - + " resources=" + resources - + " realResources=" + realResources - + " ipAddresses=" + ipAddresses - + " additionalIpAddresses=" + additionalIpAddresses - + " reports=" + reports - + " events=" + events - + " parentHostname=" + parentHostname - + " archiveUri=" + archiveUri - + " exclusiveTo=" + exclusiveTo - + " trustStore=" + trustStore - + " wireguardPubkey=" + wireguardKeyWithTimestamp - + " wantToRebuild=" + wantToRebuild - + " }"; - } - - public static class Builder { - private String hostname; - private String id; - private NodeState state; - private NodeType type; - private CloudAccount cloudAccount = CloudAccount.empty; - private String flavor; - private Optional<DockerImage> wantedDockerImage = Optional.empty(); - private Optional<DockerImage> currentDockerImage = Optional.empty(); - private Optional<Version> wantedVespaVersion = Optional.empty(); - private Optional<Version> currentVespaVersion = Optional.empty(); - private Optional<Version> wantedOsVersion = Optional.empty(); - private Optional<Version> currentOsVersion = Optional.empty(); - private OrchestratorStatus orchestratorStatus = OrchestratorStatus.NO_REMARKS; - private Optional<ApplicationId> owner = Optional.empty(); - private Optional<NodeMembership> membership = Optional.empty(); - private Optional<Long> wantedRestartGeneration = Optional.empty(); - private Optional<Long> currentRestartGeneration = Optional.empty(); - private long wantedRebootGeneration; - private long currentRebootGeneration; - private Optional<Instant> wantedFirmwareCheck = Optional.empty(); - private Optional<Instant> currentFirmwareCheck = Optional.empty(); - private Optional<String> modelName = Optional.empty(); - private NodeResources resources; - private NodeResources realResources; - private Set<String> ipAddresses = Set.of(); - private Set<String> additionalIpAddresses = Set.of(); - private NodeReports reports = new NodeReports(); - private List<Event> events = List.of(); - private Optional<String> parentHostname = Optional.empty(); - private Optional<URI> archiveUri = Optional.empty(); - private Optional<ApplicationId> exclusiveTo = Optional.empty(); - private List<TrustStoreItem> trustStore = List.of(); - private Optional<WireguardKeyWithTimestamp> wireguardPubkey = Optional.empty(); - private boolean wantToRebuild = false; - - public Builder() {} - - public Builder(NodeSpec node) { - hostname(node.hostname); - id(node.id); - state(node.state); - type(node.type); - flavor(node.flavor); - resources(node.resources); - realResources(node.realResources); - ipAddresses(node.ipAddresses); - additionalIpAddresses(node.additionalIpAddresses); - wantedRebootGeneration(node.wantedRebootGeneration); - currentRebootGeneration(node.currentRebootGeneration); - orchestratorStatus(node.orchestratorStatus); - reports(new NodeReports(node.reports)); - events(node.events); - node.wantedDockerImage.ifPresent(this::wantedDockerImage); - node.currentDockerImage.ifPresent(this::currentDockerImage); - node.wantedVespaVersion.ifPresent(this::wantedVespaVersion); - node.currentVespaVersion.ifPresent(this::currentVespaVersion); - node.wantedOsVersion.ifPresent(this::wantedOsVersion); - node.currentOsVersion.ifPresent(this::currentOsVersion); - node.owner.ifPresent(this::owner); - node.membership.ifPresent(this::membership); - node.wantedRestartGeneration.ifPresent(this::wantedRestartGeneration); - node.currentRestartGeneration.ifPresent(this::currentRestartGeneration); - node.wantedFirmwareCheck.ifPresent(this::wantedFirmwareCheck); - node.currentFirmwareCheck.ifPresent(this::currentFirmwareCheck); - node.parentHostname.ifPresent(this::parentHostname); - node.archiveUri.ifPresent(this::archiveUri); - node.exclusiveTo.ifPresent(this::exclusiveTo); - trustStore(node.trustStore); - node.wireguardKeyWithTimestamp.ifPresent(this::wireguardKeyWithTimestamp); - wantToRebuild(node.wantToRebuild); - } - - public Builder hostname(String hostname) { - this.hostname = hostname; - return this; - } - - public Builder id(String id) { - this.id = id; - return this; - } - - public Builder wantedDockerImage(DockerImage wantedDockerImage) { - this.wantedDockerImage = Optional.of(wantedDockerImage); - return this; - } - - public Builder currentDockerImage(DockerImage currentDockerImage) { - this.currentDockerImage = Optional.of(currentDockerImage); - return this; - } - - public Builder state(NodeState state) { - this.state = state; - return this; - } - - public Builder type(NodeType nodeType) { - this.type = nodeType; - return this; - } - - public Builder cloudAccount(CloudAccount cloudAccount) { - this.cloudAccount = cloudAccount; - return this; - } - - public Builder flavor(String flavor) { - this.flavor = flavor; - return this; - } - - public Builder wantedVespaVersion(Version wantedVespaVersion) { - this.wantedVespaVersion = Optional.of(wantedVespaVersion); - return this; - } - - public Builder currentVespaVersion(Version vespaVersion) { - this.currentVespaVersion = Optional.of(vespaVersion); - return this; - } - - public Builder wantedOsVersion(Version wantedOsVersion) { - this.wantedOsVersion = Optional.of(wantedOsVersion); - return this; - } - - public Builder currentOsVersion(Version currentOsVersion) { - this.currentOsVersion = Optional.of(currentOsVersion); - return this; - } - - public Builder orchestratorStatus(OrchestratorStatus orchestratorStatus) { - this.orchestratorStatus = orchestratorStatus; - return this; - } - - public Builder owner(ApplicationId owner) { - this.owner = Optional.of(owner); - return this; - } - - public Builder membership(NodeMembership membership) { - this.membership = Optional.of(membership); - return this; - } - - public Builder wantedRestartGeneration(long wantedRestartGeneration) { - this.wantedRestartGeneration = Optional.of(wantedRestartGeneration); - return this; - } - - public Builder currentRestartGeneration(long currentRestartGeneration) { - this.currentRestartGeneration = Optional.of(currentRestartGeneration); - return this; - } - - public Builder wantedRebootGeneration(long wantedRebootGeneration) { - this.wantedRebootGeneration = wantedRebootGeneration; - return this; - } - - public Builder currentRebootGeneration(long currentRebootGeneration) { - this.currentRebootGeneration = currentRebootGeneration; - return this; - } - - public Builder wantedFirmwareCheck(Instant wantedFirmwareCheck) { - this.wantedFirmwareCheck = Optional.of(wantedFirmwareCheck); - return this; - } - - public Builder currentFirmwareCheck(Instant currentFirmwareCheck) { - this.currentFirmwareCheck = Optional.of(currentFirmwareCheck); - return this; - } - - public Builder resources(NodeResources resources) { - this.resources = resources; - return this; - } - - public Builder realResources(NodeResources realResources) { - this.realResources = realResources; - return this; - } - - public Builder vcpu(double vcpu) { - return realResources(realResources.withVcpu(vcpu)); - } - - public Builder memoryGb(double memoryGb) { - return realResources(realResources.withMemoryGb(memoryGb)); - } - - public Builder diskGb(double diskGb) { - return realResources(realResources.withDiskGb(diskGb)); - } - - public Builder fastDisk(boolean fastDisk) { - return realResources(realResources.with(fastDisk ? fast : slow)); - } - - public Builder bandwidthGbps(double bandwidthGbps) { - return realResources(realResources.withBandwidthGbps(bandwidthGbps)); - } - - public Builder ipAddresses(Set<String> ipAddresses) { - this.ipAddresses = ipAddresses; - return this; - } - - public Builder additionalIpAddresses(Set<String> additionalIpAddresses) { - this.additionalIpAddresses = additionalIpAddresses; - return this; - } - - public Builder reports(NodeReports reports) { - this.reports = reports; - return this; - } - - public Builder report(String reportId, JsonNode report) { - this.reports.setReport(reportId, report); - return this; - } - - public Builder removeReport(String reportId) { - reports.removeReport(reportId); - return this; - } - - public Builder events(List<Event> events) { - this.events = events; - return this; - } - - public Builder parentHostname(String parentHostname) { - this.parentHostname = Optional.of(parentHostname); - return this; - } - - public Builder archiveUri(URI archiveUri) { - this.archiveUri = Optional.of(archiveUri); - return this; - } - - public Builder exclusiveTo(ApplicationId applicationId) { - this.exclusiveTo = Optional.of(applicationId); - return this; - } - - public Builder trustStore(List<TrustStoreItem> trustStore) { - this.trustStore = List.copyOf(trustStore); - return this; - } - - public Builder wireguardPubkey(WireguardKey wireguardPubkey) { - this.wireguardPubkey = Optional.of(new WireguardKeyWithTimestamp(wireguardPubkey, Instant.EPOCH)); - return this; - } - - public Builder wireguardKeyWithTimestamp(WireguardKeyWithTimestamp wireguardPubKey) { - this.wireguardPubkey = Optional.of(wireguardPubKey); - return this; - } - - public Builder wantToRebuild(boolean wantToRebuild) { - this.wantToRebuild = wantToRebuild; - return this; - } - - public Builder updateFromNodeAttributes(NodeAttributes attributes) { - attributes.getHostId().ifPresent(this::id); - attributes.getDockerImage().ifPresent(this::currentDockerImage); - attributes.getCurrentOsVersion().ifPresent(this::currentOsVersion); - attributes.getRebootGeneration().ifPresent(this::currentRebootGeneration); - attributes.getRestartGeneration().ifPresent(this::currentRestartGeneration); - // Always replace entire trust store - trustStore(attributes.getTrustStore()); - attributes.getWireguardPubkey().ifPresent(this::wireguardPubkey); - this.reports.updateFromRawMap(attributes.getReports()); - - return this; - } - - public String hostname() { - return hostname; - } - - public Optional<DockerImage> wantedDockerImage() { - return wantedDockerImage; - } - - public Optional<DockerImage> currentDockerImage() { - return currentDockerImage; - } - - public NodeState state() { - return state; - } - - public NodeType type() { - return type; - } - - public CloudAccount cloudAccount() { - return cloudAccount; - } - - public String flavor() { - return flavor; - } - - public Optional<Version> wantedVespaVersion() { - return wantedVespaVersion; - } - - public Optional<Version> currentVespaVersion() { - return currentVespaVersion; - } - - public Optional<Version> wantedOsVersion() { - return wantedOsVersion; - } - - public Optional<Version> currentOsVersion() { - return currentOsVersion; - } - - public OrchestratorStatus orchestratorStatus() { - return orchestratorStatus; - } - - public Optional<ApplicationId> owner() { - return owner; - } - - public Optional<NodeMembership> membership() { - return membership; - } - - public Optional<Long> wantedRestartGeneration() { - return wantedRestartGeneration; - } - - public Optional<Long> currentRestartGeneration() { - return currentRestartGeneration; - } - - public long wantedRebootGeneration() { - return wantedRebootGeneration; - } - - public long currentRebootGeneration() { - return currentRebootGeneration; - } - - public NodeResources resources() { - return resources; - } - - public NodeResources realResources() { - return realResources; - } - - public Set<String> ipAddresses() { - return ipAddresses; - } - - public Set<String> additionalIpAddresses() { - return additionalIpAddresses; - } - - public NodeReports reports() { - return reports; - } - - public List<Event> events() { - return events; - } - - public Optional<String> parentHostname() { - return parentHostname; - } - - public Optional<URI> archiveUri() { - return archiveUri; - } - - public NodeSpec build() { - return new NodeSpec(hostname, id, wantedDockerImage, currentDockerImage, state, type, cloudAccount, flavor, - wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, orchestratorStatus, - owner, membership, - wantedRestartGeneration, currentRestartGeneration, - wantedRebootGeneration, currentRebootGeneration, - wantedFirmwareCheck, currentFirmwareCheck, modelName, - resources, realResources, ipAddresses, additionalIpAddresses, - reports, events, parentHostname, archiveUri, exclusiveTo, trustStore, - wireguardPubkey, wantToRebuild); - } - - - public static Builder testSpec(String hostname) { - return testSpec(hostname, NodeState.active); - } - - /** - * Creates a NodeSpec.Builder that has the given hostname, in a given state, and some - * reasonable values for the remaining required NodeSpec fields. - */ - public static Builder testSpec(String hostname, NodeState state) { - Builder builder = new Builder() - .id(hostname) - .hostname(hostname) - .state(state) - .type(NodeType.tenant) - .flavor("d-2-8-50") - .resources(new NodeResources(2, 8, 50, 10)) - .realResources(new NodeResources(2, 8, 50, 10)) - .events(List.of(new Event("operator", "rebooted", Instant.EPOCH))); - - // Set the required allocated fields - if (EnumSet.of(NodeState.active, NodeState.inactive, NodeState.reserved).contains(state)) { - builder .owner(ApplicationId.defaultId()) - .membership(new NodeMembership("container", "my-id", "group", 0, false)) - .wantedVespaVersion(Version.fromString("7.1.1")) - .wantedDockerImage(DockerImage.fromString("docker.domain.tld/repo/image:7.1.1")) - .currentRestartGeneration(0) - .wantedRestartGeneration(0); - } - - return builder; - } - } - - private static void requireOptional(Optional<?> optional, String name) { - if (optional == null || optional.isEmpty()) - throw new IllegalArgumentException(name + " must be set, was " + optional); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java deleted file mode 100644 index 8e66480c92a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -/** - * All the states a node can be in the node-repository. - * - * See com.yahoo.vespa.hosted.provision.NodeState - * - * @author freva - */ -public enum NodeState { - provisioned, ready, reserved, active, inactive, dirty, failed, parked, deprovisioned, breakfixed -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/OrchestratorStatus.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/OrchestratorStatus.java deleted file mode 100644 index d8532188c64..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/OrchestratorStatus.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import java.util.stream.Stream; - -public enum OrchestratorStatus { - NO_REMARKS, ALLOWED_TO_BE_DOWN, PERMANENTLY_DOWN, UNKNOWN; - - public static OrchestratorStatus fromString(String statusString) { - return Stream.of(values()) - .filter(status -> status.asString().equals(statusString)) - .findFirst() - .orElse(UNKNOWN); - } - - public String asString() { - return name(); - } - - public boolean isSuspended() { - return this != NO_REMARKS; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java deleted file mode 100644 index d340aa9fd3d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ /dev/null @@ -1,406 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.net.InetAddresses; -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.config.provision.WireguardKeyWithTimestamp; -import com.yahoo.config.provision.host.FlavorOverrides; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; -import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetAclResponse; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetNodesResponse; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetWireguardResponse; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeRepositoryNode; -import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; - -import java.net.URI; -import java.time.Instant; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeMap; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author stiankri - * @author dybis - */ -public class RealNodeRepository implements NodeRepository { - private static final Logger logger = Logger.getLogger(RealNodeRepository.class.getName()); - - private final ConfigServerApi configServerApi; - - public RealNodeRepository(ConfigServerApi configServerApi) { - this.configServerApi = configServerApi; - } - - @Override - public void addNodes(List<AddNode> nodes) { - List<NodeRepositoryNode> nodesToPost = nodes.stream() - .map(RealNodeRepository::nodeRepositoryNodeFromAddNode) - .toList(); - - configServerApi.post("/nodes/v2/node", nodesToPost, StandardConfigServerResponse.class) - .throwOnError("Failed to add nodes"); - } - - @Override - public List<NodeSpec> getNodes(String baseHostName) { - String path = "/nodes/v2/node/?recursive=true&parentHost=" + baseHostName; - final GetNodesResponse nodesForHost = configServerApi.get(path, GetNodesResponse.class); - - return nodesForHost.nodes.stream() - .map(RealNodeRepository::createNodeSpec) - .toList(); - } - - @Override - public Optional<NodeSpec> getOptionalNode(String hostName) { - try { - NodeRepositoryNode nodeResponse = configServerApi.get("/nodes/v2/node/" + hostName, - NodeRepositoryNode.class); - - return Optional.ofNullable(nodeResponse).map(RealNodeRepository::createNodeSpec); - } catch (HttpException.NotFoundException | HttpException.ForbiddenException e) { - // Return empty on 403 in addition to 404 as it likely means we're trying to access a node that - // has been deleted. When a node is deleted, the parent-child relationship no longer exists and - // authorization cannot be granted. - return Optional.empty(); - } - } - - /** - * Get all ACLs that belongs to a hostname. Usually this is a parent host and all - * ACLs for child nodes are returned. - */ - @Override - public Map<String, Acl> getAcls(String hostName) { - String path = String.format("/nodes/v2/acl/%s?children=true", hostName); - GetAclResponse response = configServerApi.get(path, GetAclResponse.class); - - // Group ports by container hostname that trusts them - Map<String, Set<Integer>> trustedPorts = response.trustedPorts.stream() - .collect(Collectors.groupingBy( - GetAclResponse.Port::getTrustedBy, - Collectors.mapping(port -> port.port, Collectors.toSet()))); - - // Group UDP ports by container hostname that trusts them - Map<String, Set<Integer>> trustedUdpPorts = response.trustedUdpPorts.stream() - .collect(Collectors.groupingBy( - GetAclResponse.Port::getTrustedBy, - Collectors.mapping(port -> port.port, Collectors.toSet()))); - - // Group node ip-addresses by container hostname that trusts them - Map<String, Set<Acl.Node>> trustedNodes = response.trustedNodes.stream() - .collect(Collectors.groupingBy( - GetAclResponse.Node::getTrustedBy, - Collectors.mapping( - node -> new Acl.Node(node.hostname, node.ipAddress, Set.copyOf(node.ports)), - Collectors.toSet()))); - - // Group trusted networks by container hostname that trusts them - Map<String, Set<String>> trustedNetworks = response.trustedNetworks.stream() - .collect(Collectors.groupingBy(GetAclResponse.Network::getTrustedBy, - Collectors.mapping(node -> node.network, Collectors.toSet()))); - - - // For each hostname create an ACL - return Stream.of(trustedNodes.keySet(), trustedPorts.keySet(), trustedUdpPorts.keySet(), trustedNetworks.keySet()) - .flatMap(Set::stream) - .distinct() - .collect(Collectors.toMap( - Function.identity(), - hostname -> new Acl(trustedPorts.get(hostname), - trustedUdpPorts.get(hostname), - trustedNodes.get(hostname), - trustedNetworks.get(hostname)))); - } - - @Override - public List<WireguardPeer> getExclavePeers() { - String path = "/nodes/v2/node/?recursive=true&enclave=true"; - final GetNodesResponse response = configServerApi.get(path, GetNodesResponse.class); - - return response.nodes.stream() - .mapMulti((NodeRepositoryNode node, Consumer<WireguardPeer> consumer) -> { - var keyWithTimestamp = createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp, - node.wireguardPubkey, - node.wireguardKeyTimestamp); - if (keyWithTimestamp == null) return; - - List<VersionedIpAddress> ipAddresses = getIpAddresses(node); - if (ipAddresses.isEmpty()) return; - - consumer.accept(new WireguardPeer(HostName.of(node.hostname), ipAddresses, keyWithTimestamp)); - }) - .sorted() - .toList(); - } - - private static List<VersionedIpAddress> getIpAddresses(NodeRepositoryNode node) { - return node.ipAddresses.stream() - .map(InetAddresses::forString) - .filter(address -> !address.isLoopbackAddress() && !address.isLinkLocalAddress() && !address.isSiteLocalAddress()) - .map(VersionedIpAddress::from) - .toList(); - } - - @Override - public List<WireguardPeer> getConfigserverPeers() { - GetWireguardResponse response = configServerApi.get("/nodes/v2/wireguard", GetWireguardResponse.class); - return response.configservers.stream() - .map(RealNodeRepository::createConfigserverPeer) - .sorted(Comparator.comparing(WireguardPeer::hostname)) - .toList(); - } - - @Override - public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) { - configServerApi.patch("/nodes/v2/node/" + hostName, - nodeRepositoryNodeFromNodeAttributes(nodeAttributes), - StandardConfigServerResponse.class) - .throwOnError("Failed to update node attributes"); - } - - @Override - public void setNodeState(String hostName, NodeState nodeState) { - String state = nodeState.name(); - StandardConfigServerResponse response = configServerApi.put("/nodes/v2/state/" + state + "/" + hostName, - Optional.empty(), /* body */ - StandardConfigServerResponse.class); - logger.info(response.message); - response.throwOnError("Failed to set node state"); - } - - @Override - public void reboot(String hostname) { - String uri = "/nodes/v2/command/reboot?hostname=" + hostname; - StandardConfigServerResponse response = configServerApi.post(uri, Optional.empty(), StandardConfigServerResponse.class); - logger.info(response.message); - response.throwOnError("Failed to reboot " + hostname); - } - - private static NodeSpec createNodeSpec(NodeRepositoryNode node) { - Objects.requireNonNull(node.type, "Unknown node type"); - NodeType nodeType = NodeType.valueOf(node.type); - - Objects.requireNonNull(node.state, "Unknown node state"); - NodeState nodeState = NodeState.valueOf(node.state); - - Optional<NodeMembership> membership = Optional.ofNullable(node.membership) - .map(m -> new NodeMembership(m.clusterType, m.clusterId, m.group, m.index, m.retired)); - NodeReports reports = NodeReports.fromMap(Optional.ofNullable(node.reports).orElseGet(Map::of)); - List<Event> events = node.history.stream() - .map(event -> new Event(event.agent, event.event, Optional.ofNullable(event.at).map(Instant::ofEpochMilli).orElse(Instant.EPOCH))) - .toList(); - - List<TrustStoreItem> trustStore = Optional.ofNullable(node.trustStore).orElse(List.of()).stream() - .map(item -> new TrustStoreItem(item.fingerprint, Instant.ofEpochMilli(item.expiry))) - .toList(); - - - return new NodeSpec( - node.hostname, - node.id, - Optional.ofNullable(node.wantedDockerImage).map(DockerImage::fromString), - Optional.ofNullable(node.currentDockerImage).map(DockerImage::fromString), - nodeState, - nodeType, - Optional.ofNullable(node.cloudAccount).map(CloudAccount::from).orElse(CloudAccount.empty), - node.flavor, - Optional.ofNullable(node.wantedVespaVersion).map(Version::fromString), - Optional.ofNullable(node.vespaVersion).map(Version::fromString), - Optional.ofNullable(node.wantedOsVersion).map(Version::fromString), - Optional.ofNullable(node.currentOsVersion).map(Version::fromString), - Optional.ofNullable(node.orchestratorStatus).map(OrchestratorStatus::fromString).orElse(OrchestratorStatus.NO_REMARKS), - Optional.ofNullable(node.owner).map(o -> ApplicationId.from(o.tenant, o.application, o.instance)), - membership, - Optional.ofNullable(node.restartGeneration), - Optional.ofNullable(node.currentRestartGeneration), - node.rebootGeneration, - node.currentRebootGeneration, - Optional.ofNullable(node.wantedFirmwareCheck).map(Instant::ofEpochMilli), - Optional.ofNullable(node.currentFirmwareCheck).map(Instant::ofEpochMilli), - Optional.ofNullable(node.modelName), - nodeResources(node.resources), - nodeResources(node.realResources), - node.ipAddresses, - node.additionalIpAddresses, - reports, - events, - Optional.ofNullable(node.parentHostname), - Optional.ofNullable(node.archiveUri).map(URI::create), - Optional.ofNullable(node.exclusiveTo).map(ApplicationId::fromSerializedForm), - trustStore, - Optional.ofNullable(createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp, - node.wireguardPubkey, - node.wireguardKeyTimestamp)), - node.wantToRebuild); - } - - private static NodeResources nodeResources(NodeRepositoryNode.NodeResources nodeResources) { - return new NodeResources( - nodeResources.vcpu, - nodeResources.memoryGb, - nodeResources.diskGb, - nodeResources.bandwidthGbps, - diskSpeedFromString(nodeResources.diskSpeed), - storageTypeFromString(nodeResources.storageType), - architectureFromString(nodeResources.architecture), - gpuResourcesFrom(nodeResources)); - } - - private static NodeResources.GpuResources gpuResourcesFrom(NodeRepositoryNode.NodeResources nodeResources) { - if (nodeResources.gpuCount == null || nodeResources.gpuMemoryGb == null) return NodeResources.GpuResources.zero(); - return new NodeResources.GpuResources(nodeResources.gpuCount, nodeResources.gpuMemoryGb); - } - - private static NodeResources.DiskSpeed diskSpeedFromString(String diskSpeed) { - if (diskSpeed == null) return NodeResources.DiskSpeed.getDefault(); - return switch (diskSpeed) { - case "fast" -> NodeResources.DiskSpeed.fast; - case "slow" -> NodeResources.DiskSpeed.slow; - case "any" -> NodeResources.DiskSpeed.any; - default -> throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); - }; - } - - private static NodeResources.StorageType storageTypeFromString(String storageType) { - if (storageType == null) return NodeResources.StorageType.getDefault(); - return switch (storageType) { - case "remote" -> NodeResources.StorageType.remote; - case "local" -> NodeResources.StorageType.local; - case "any" -> NodeResources.StorageType.any; - default -> throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); - }; - } - - private static NodeResources.Architecture architectureFromString(String architecture) { - if (architecture == null) return NodeResources.Architecture.getDefault(); - return switch (architecture) { - case "arm64" -> NodeResources.Architecture.arm64; - case "x86_64" -> NodeResources.Architecture.x86_64; - case "any" -> NodeResources.Architecture.any; - default -> throw new IllegalArgumentException("Unknown architecture '" + architecture + "'"); - }; - } - - private static String toString(NodeResources.DiskSpeed diskSpeed) { - return switch (diskSpeed) { - case fast -> "fast"; - case slow -> "slow"; - case any -> "any"; - }; - } - - private static String toString(NodeResources.StorageType storageType) { - return switch (storageType) { - case remote -> "remote"; - case local -> "local"; - case any -> "any"; - }; - } - - private static String toString(NodeResources.Architecture architecture) { - return switch (architecture) { - case arm64 -> "arm64"; - case x86_64 -> "x86_64"; - case any -> "any"; - }; - } - - private static NodeRepositoryNode nodeRepositoryNodeFromAddNode(AddNode addNode) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.id = addNode.id; - node.hostname = addNode.hostname; - node.parentHostname = addNode.parentHostname.orElse(null); - addNode.nodeFlavor.ifPresent(f -> node.flavor = f); - addNode.flavorOverrides.flatMap(FlavorOverrides::diskGb).ifPresent(d -> { - node.resources = new NodeRepositoryNode.NodeResources(); - node.resources.diskGb = d; - }); - addNode.nodeResources.ifPresent(resources -> { - node.resources = new NodeRepositoryNode.NodeResources(); - node.resources.vcpu = resources.vcpu(); - node.resources.memoryGb = resources.memoryGb(); - node.resources.diskGb = resources.diskGb(); - node.resources.bandwidthGbps = resources.bandwidthGbps(); - node.resources.diskSpeed = toString(resources.diskSpeed()); - node.resources.storageType = toString(resources.storageType()); - node.resources.architecture = toString(resources.architecture()); - if (!resources.gpuResources().isZero()) { - node.resources.gpuCount = resources.gpuResources().count(); - node.resources.gpuMemoryGb = resources.gpuResources().memoryGb(); - } - }); - node.type = addNode.nodeType.name(); - node.ipAddresses = addNode.ipAddresses; - node.additionalIpAddresses = addNode.additionalIpAddresses; - return node; - } - - public static NodeRepositoryNode nodeRepositoryNodeFromNodeAttributes(NodeAttributes nodeAttributes) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.id = nodeAttributes.getHostId().orElse(null); - node.currentDockerImage = nodeAttributes.getDockerImage().map(DockerImage::asString).orElse(null); - node.currentRestartGeneration = nodeAttributes.getRestartGeneration().orElse(null); - node.currentRebootGeneration = nodeAttributes.getRebootGeneration().orElse(null); - node.vespaVersion = nodeAttributes.getVespaVersion().map(Version::toFullString).orElse(null); - node.currentOsVersion = nodeAttributes.getCurrentOsVersion().map(Version::toFullString).orElse(null); - node.currentFirmwareCheck = nodeAttributes.getCurrentFirmwareCheck().map(Instant::toEpochMilli).orElse(null); - node.trustStore = nodeAttributes.getTrustStore().stream() - .map(item -> new NodeRepositoryNode.TrustStoreItem(item.fingerprint(), item.expiry().toEpochMilli())) - .toList(); - // This is used for patching, and timestamp must only be set on the server side, hence sending EPOCH. - node.wireguardKeyWithTimestamp = nodeAttributes.getWireguardPubkey() - .map(key -> new NodeRepositoryNode.WireguardKeyWithTimestamp(key.value(), 0L)) - .orElse(null); - Map<String, JsonNode> reports = nodeAttributes.getReports(); - node.reports = reports == null || reports.isEmpty() ? null : new TreeMap<>(reports); - - // TODO wg: remove when all nodes are using new key+timestamp format - node.wireguardPubkey = nodeAttributes.getWireguardPubkey().map(WireguardKey::value).orElse(null); - return node; - } - - private static WireguardPeer createConfigserverPeer(GetWireguardResponse.Configserver configServer) { - return new WireguardPeer(HostName.of(configServer.hostname), - configServer.ipAddresses.stream().map(VersionedIpAddress::from).toList(), - createWireguardKeyWithTimestamp(configServer.wireguardKeyWithTimestamp, - configServer.wireguardPubkey, - configServer.wireguardKeyTimestamp)); - } - - private static WireguardKeyWithTimestamp createWireguardKeyWithTimestamp(NodeRepositoryNode.WireguardKeyWithTimestamp wirguardJson, - String oldKeyJson, Long oldTimestampJson) { - if (wirguardJson != null && wirguardJson.key != null && ! wirguardJson.key.isEmpty()) { - return new WireguardKeyWithTimestamp(WireguardKey.from(wirguardJson.key), - Instant.ofEpochMilli(wirguardJson.timestamp)); - // TODO wg: remove when all nodes are using new key+timestamp format - } else if (oldKeyJson != null) { - var timestamp = oldTimestampJson != null ? oldTimestampJson : 0L; - return new WireguardKeyWithTimestamp(WireguardKey.from(oldKeyJson), - Instant.ofEpochMilli(timestamp)); - // TODO END - } else return null; - - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java deleted file mode 100644 index dfec70288e3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/TrustStoreItem.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import java.time.Instant; -import java.util.Objects; - -/** - * @author mortent - */ -public class TrustStoreItem { - private final String fingerprint; - private final Instant expiry; - - public TrustStoreItem(String fingerprint, Instant expiry) { - this.fingerprint = fingerprint; - this.expiry = expiry; - } - - public String fingerprint() { - return fingerprint; - } - - public Instant expiry() { - return expiry; - } - - @Override - public String toString() { - return "TrustStoreItem{" + - "fingerprint='" + fingerprint + '\'' + - ", expiry=" + expiry + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TrustStoreItem that = (TrustStoreItem) o; - return Objects.equals(fingerprint, that.fingerprint) && Objects.equals(expiry, that.expiry); - } - - @Override - public int hashCode() { - return Objects.hash(fingerprint, expiry); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java deleted file mode 100644 index d20f31e256e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * This class represents a response from the /nodes/v2/acl/ API. - * - * @author mpolden - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class GetAclResponse { - - @JsonProperty("trustedNodes") - public final List<Node> trustedNodes; - - @JsonProperty("trustedNetworks") - public final List<Network> trustedNetworks; - - @JsonProperty("trustedPorts") - public final List<Port> trustedPorts; - - @JsonProperty("trustedUdpPorts") - public final List<Port> trustedUdpPorts; - - @JsonCreator - public GetAclResponse(@JsonProperty("trustedNodes") List<Node> trustedNodes, - @JsonProperty("trustedNetworks") List<Network> trustedNetworks, - @JsonProperty("trustedPorts") List<Port> trustedPorts, - @JsonProperty("trustedUdpPorts") List<Port> trustedUdpPorts) { - this.trustedNodes = trustedNodes == null ? List.of() : List.copyOf(trustedNodes); - this.trustedNetworks = trustedNetworks == null ? List.of() : List.copyOf(trustedNetworks); - this.trustedPorts = trustedPorts == null ? List.of() : List.copyOf(trustedPorts); - this.trustedUdpPorts = trustedUdpPorts == null ? List.of() : List.copyOf(trustedUdpPorts); - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Node { - - @JsonProperty("hostname") - public final String hostname; - - @JsonProperty("type") - public final String nodeType; - - @JsonProperty("ipAddress") - public final String ipAddress; - - @JsonProperty("ports") - public final List<Integer> ports; - - @JsonProperty("trustedBy") - public final String trustedBy; - - @JsonCreator - public Node(@JsonProperty("hostname") String hostname, @JsonProperty("type") String nodeType, - @JsonProperty("ipAddress") String ipAddress, @JsonProperty("ports") List<Integer> ports, - @JsonProperty("trustedBy") String trustedBy) { - this.hostname = hostname; - this.nodeType = nodeType; - this.ipAddress = ipAddress; - this.ports = ports == null ? List.of() : List.copyOf(ports); - this.trustedBy = trustedBy; - } - - public String getTrustedBy() { - return trustedBy; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Network { - - @JsonProperty("network") - public final String network; - - @JsonProperty("trustedBy") - public final String trustedBy; - - @JsonCreator - public Network(@JsonProperty("network") String network, @JsonProperty("trustedBy") String trustedBy) { - this.network = network; - this.trustedBy = trustedBy; - } - - public String getTrustedBy() { - return trustedBy; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Port { - - @JsonProperty("port") - public final Integer port; - - @JsonProperty("trustedBy") - public final String trustedBy; - - @JsonCreator - public Port(@JsonProperty("port") Integer port, @JsonProperty("trustedBy") String trustedBy) { - this.port = port; - this.trustedBy = trustedBy; - } - - public String getTrustedBy() { - return trustedBy; - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java deleted file mode 100644 index b744c935247..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Collections; -import java.util.List; - -/** - * This class represents a response from the /nodes/v2/node/ API. It is designed to be - * usable by any module, by not depending itself on any module-specific classes. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class GetNodesResponse { - - public final List<NodeRepositoryNode> nodes; - - @JsonCreator - public GetNodesResponse(@JsonProperty("nodes") List<NodeRepositoryNode> nodes) { - this.nodes = Collections.unmodifiableList(nodes); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java deleted file mode 100644 index 572323d733b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * A response from the /nodes/v2/wireguard api. - * - * @author gjoranv - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class GetWireguardResponse { - - public final List<Configserver> configservers; - - @JsonCreator - public GetWireguardResponse(@JsonProperty("configservers") List<Configserver> configservers) { - this.configservers = configservers; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Configserver { - - @JsonProperty("hostname") - public String hostname; - - @JsonProperty("ipAddresses") - public List<String> ipAddresses; - - @JsonProperty("wireguard") - public NodeRepositoryNode.WireguardKeyWithTimestamp wireguardKeyWithTimestamp; - - - // TODO wg: remove when all nodes use new key+timestamp format - @JsonProperty("wireguardPubkey") - @JsonInclude(JsonInclude.Include.NON_EMPTY) - public String wireguardPubkey; - @JsonProperty("wireguardKeyTimestamp") - @JsonInclude(JsonInclude.Include.NON_EMPTY) - public Long wireguardKeyTimestamp; - - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java deleted file mode 100644 index c377d521648..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonNode; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * @author freva - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class NodeRepositoryNode { - - @JsonProperty("state") - public String state; - @JsonProperty("hostname") - public String hostname; - @JsonProperty("ipAddresses") - public Set<String> ipAddresses; - @JsonProperty("additionalIpAddresses") - public Set<String> additionalIpAddresses; - @JsonProperty("id") - public String id; - @JsonProperty("cloudAccount") - public String cloudAccount; - @JsonProperty("flavor") - public String flavor; - @JsonProperty("resources") - public NodeResources resources; - @JsonProperty("realResources") - public NodeResources realResources; - @JsonProperty("membership") - public Membership membership; - @JsonProperty("owner") - public Owner owner; - @JsonProperty("restartGeneration") - public Long restartGeneration; - @JsonProperty("rebootGeneration") - public Long rebootGeneration; - @JsonProperty("currentRestartGeneration") - public Long currentRestartGeneration; - @JsonProperty("currentRebootGeneration") - public Long currentRebootGeneration; - @JsonProperty("vespaVersion") - public String vespaVersion; - @JsonProperty("wantedVespaVersion") - public String wantedVespaVersion; - @JsonProperty("currentOsVersion") - public String currentOsVersion; - @JsonProperty("wantedOsVersion") - public String wantedOsVersion; - @JsonProperty("currentFirmwareCheck") - public Long currentFirmwareCheck; - @JsonProperty("wantedFirmwareCheck") - public Long wantedFirmwareCheck; - @JsonProperty("modelName") - public String modelName; - @JsonProperty("failCount") - public Integer failCount; - @JsonProperty("environment") - public String environment; - @JsonProperty("reservedTo") - public String reservedTo; - @JsonProperty("type") - public String type; - @JsonProperty("wantedDockerImage") - public String wantedDockerImage; - @JsonProperty("currentDockerImage") - public String currentDockerImage; - @JsonProperty("parentHostname") - public String parentHostname; - @JsonProperty("wantToRetire") - public Boolean wantToRetire; - @JsonProperty("wantToDeprovision") - public Boolean wantToDeprovision; - @JsonProperty("wantToRebuild") - public Boolean wantToRebuild; - @JsonProperty("orchestratorStatus") - public String orchestratorStatus; - @JsonProperty("archiveUri") - public String archiveUri; - @JsonProperty("exclusiveTo") - public String exclusiveTo; - @JsonProperty("history") - public List<Event> history; - @JsonProperty("trustStore") - @JsonInclude(JsonInclude.Include.NON_EMPTY) - public List<TrustStoreItem> trustStore; - @JsonProperty("wireguard") - public WireguardKeyWithTimestamp wireguardKeyWithTimestamp; - - // TODO wg: remove separate key and timestamp when all nodes use new keyWithTimestamp - @JsonProperty("wireguardPubkey") - @JsonInclude(JsonInclude.Include.NON_EMPTY) - public String wireguardPubkey; - @JsonProperty("wireguardKeyTimestamp") - @JsonInclude(JsonInclude.Include.NON_EMPTY) - public Long wireguardKeyTimestamp; - - @JsonProperty("reports") - public Map<String, JsonNode> reports = null; - - @Override - public String toString() { - return "NodeRepositoryNode{" + - "state='" + state + '\'' + - ", hostname='" + hostname + '\'' + - ", ipAddresses=" + ipAddresses + - ", additionalIpAddresses=" + additionalIpAddresses + - ", id='" + id + '\'' + - ", flavor='" + flavor + '\'' + - ", resources=" + resources + - ", realResources=" + realResources + - ", membership=" + membership + - ", owner=" + owner + - ", restartGeneration=" + restartGeneration + - ", rebootGeneration=" + rebootGeneration + - ", currentRestartGeneration=" + currentRestartGeneration + - ", currentRebootGeneration=" + currentRebootGeneration + - ", vespaVersion='" + vespaVersion + '\'' + - ", wantedVespaVersion='" + wantedVespaVersion + '\'' + - ", currentOsVersion='" + currentOsVersion + '\'' + - ", wantedOsVersion='" + wantedOsVersion + '\'' + - ", currentFirmwareCheck=" + currentFirmwareCheck + - ", wantedFirmwareCheck=" + wantedFirmwareCheck + - ", modelName='" + modelName + '\'' + - ", failCount=" + failCount + - ", environment='" + environment + '\'' + - ", reservedTo='" + reservedTo + '\'' + - ", type='" + type + '\'' + - ", wantedDockerImage='" + wantedDockerImage + '\'' + - ", currentDockerImage='" + currentDockerImage + '\'' + - ", parentHostname='" + parentHostname + '\'' + - ", wantToRetire=" + wantToRetire + - ", wantToDeprovision=" + wantToDeprovision + - ", wantToRebuild=" + wantToRebuild + - ", orchestratorStatus='" + orchestratorStatus + '\'' + - ", archiveUri='" + archiveUri + '\'' + - ", exclusiveTo='" + exclusiveTo + '\'' + - ", history=" + history + - ", trustStore=" + trustStore + - ", wireguard=" + wireguardKeyTimestamp + - ", reports=" + reports + - '}'; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class WireguardKeyWithTimestamp { - @JsonProperty("key") - public String key; - @JsonProperty("timestamp") - public long timestamp; - - public WireguardKeyWithTimestamp(@JsonProperty("key") String key, @JsonProperty("timestamp") long timestamp) { - this.key = key; - this.timestamp = timestamp; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Owner { - @JsonProperty("tenant") - public String tenant; - @JsonProperty("application") - public String application; - @JsonProperty("instance") - public String instance; - - public String toString() { - return "Owner {" + - " tenant = " + tenant + - " application = " + application + - " instance = " + instance + - " }"; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Membership { - @JsonProperty("clustertype") - public String clusterType; - @JsonProperty("clusterid") - public String clusterId; - @JsonProperty("group") - public String group; - @JsonProperty("index") - public int index; - @JsonProperty("retired") - public boolean retired; - - @Override - public String toString() { - return "Membership {" + - " clusterType = " + clusterType + - " clusterId = " + clusterId + - " group = " + group + - " index = " + index + - " retired = " + retired + - " }"; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - @JsonInclude(JsonInclude.Include.NON_NULL) - public static class NodeResources { - @JsonProperty - public Double vcpu; - @JsonProperty - public Double memoryGb; - @JsonProperty - public Double diskGb; - @JsonProperty - public Double bandwidthGbps; - @JsonProperty - public String diskSpeed; - @JsonProperty - public String storageType; - @JsonProperty - public String architecture; - @JsonProperty - public Integer gpuCount; - @JsonProperty - public Double gpuMemoryGb; - - @Override - public String toString() { - return "NodeResources{" + - "vcpu=" + vcpu + - ", memoryGb=" + memoryGb + - ", diskGb=" + diskGb + - ", bandwidthGbps=" + bandwidthGbps + - ", diskSpeed='" + diskSpeed + '\'' + - ", storageType='" + storageType + '\'' + - ", architecture='" + architecture + '\'' + - ", gpuCount=" + gpuCount + - ", gpuMemoryGb=" + gpuMemoryGb + - '}'; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - @JsonInclude(JsonInclude.Include.NON_NULL) - public static class Event { - @JsonProperty - public String event; - @JsonProperty - public String agent; - @JsonProperty - public Long at; - - @Override - public String toString() { - return "Event{" + - "agent=" + agent + - ", event=" + event + - ", at=" + at + - '}'; - } - } - @JsonIgnoreProperties(ignoreUnknown = true) - @JsonInclude(JsonInclude.Include.NON_NULL) - public static class TrustStoreItem { - @JsonProperty ("fingerprint") - public String fingerprint; - @JsonProperty ("expiry") - public long expiry; - - public TrustStoreItem(@JsonProperty("fingerprint") String fingerprint, @JsonProperty("expiry") long expiry) { - this.fingerprint = fingerprint; - this.expiry = expiry; - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java deleted file mode 100644 index bf83a1a4bdf..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java deleted file mode 100644 index ccc1f469e1e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.stream.Stream; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * The most basic form of a node repository report on a node. - * - * <p>This class can be used directly for simple reports, or can be used as a base class for richer reports. - * - * <p><strong>Subclass requirements</strong> - * - * <ol> - * <li>A subclass must be a Jackson class that can be mapped to {@link JsonNode} with {@link #toJsonNode()}, - * and from {@link JsonNode} with {@link #fromJsonNode(JsonNode, Class)}.</li> - * <li>A subclass must override {@link #updates(BaseReport)} and make sure to return true if - * {@code super.updates(current)}.</li> - * </ol> - * - * @author hakonhall - */ -// @Immutable -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class BaseReport { - /** The time the report was created, in milliseconds since Epoch. */ - public static final String CREATED_FIELD = "createdMillis"; - /** The description of the error (implies wanting to fail out node). */ - public static final String DESCRIPTION_FIELD = "description"; - /** The type of report, see {@link Type} enum. */ - public static final String TYPE_FIELD = "type"; - - protected static final ObjectMapper mapper = new ObjectMapper(); - - private final OptionalLong createdMillis; - private final Optional<String> description; - private final Type type; - - public enum Type { - /** The default type if none given, or not recognized. */ - UNSPECIFIED, - /** A program to be executed once. */ - ONCE, - /** The host has a soft failure and should be parked for manual inspection. */ - SOFT_FAIL, - /** The host has a hard failure and should be given back to siteops. */ - HARD_FAIL; - - public static Optional<Type> deserialize(String typeString) { - return Stream.of(Type.values()).filter(type -> type.name().equalsIgnoreCase(typeString)).findAny(); - } - - public String serialize() { return name(); } - } - - @JsonCreator - public BaseReport(@JsonProperty(CREATED_FIELD) Long createdMillisOrNull, - @JsonProperty(DESCRIPTION_FIELD) String descriptionOrNull, - @JsonProperty(TYPE_FIELD) Type typeOrNull) { - this.createdMillis = createdMillisOrNull == null ? OptionalLong.empty() : OptionalLong.of(createdMillisOrNull); - this.description = Optional.ofNullable(descriptionOrNull); - this.type = typeOrNull == null ? Type.UNSPECIFIED : typeOrNull; - } - - public BaseReport(Long createdMillisOrNull, String descriptionOrNull) { - this(createdMillisOrNull, descriptionOrNull, Type.UNSPECIFIED); - } - - @JsonGetter(CREATED_FIELD) - public final Long getCreatedMillisOrNull() { - return createdMillis.isPresent() ? createdMillis.getAsLong() : null; - } - - @JsonGetter(DESCRIPTION_FIELD) - public final String getDescriptionOrNull() { - return description.orElse(null); - } - - /** null is returned on UNSPECIFIED to avoid noisy reports. */ - @JsonGetter(TYPE_FIELD) - public final Type getTypeOrNull() { - return type == Type.UNSPECIFIED ? null : type; - } - - public Type getType() { - return type; - } - - /** - * Assume {@code this} is a freshly made report, and {@code current} is the report in the node repository: - * Return true iff the node repository should be updated. - * - * <p>The createdMillis field is ignored in this method (unless it is earlier than {@code current}'s?). - */ - public boolean updates(BaseReport current) { - if (this == current) return false; - if (this.getClass() != current.getClass()) return true; - return !Objects.equals(description, current.description) || - !Objects.equals(type, current.type); - } - - /** A variant of {@link #updates(BaseReport)} handling possibly absent reports, whether new or old. */ - public static <TNEW extends BaseReport, TOLD extends BaseReport> - boolean updates2(Optional<TNEW> newReport, Optional<TOLD> oldReport) { - if (newReport.isPresent() ^ oldReport.isPresent()) return true; - return newReport.map(r -> r.updates(oldReport.get())).orElse(false); - } - - public static BaseReport fromJsonNode(JsonNode jsonNode) { - return fromJsonNode(jsonNode, BaseReport.class); - } - - public static <R extends BaseReport> R fromJsonNode(JsonNode jsonNode, Class<R> jacksonClass) { - return uncheck(() -> mapper.treeToValue(jsonNode, jacksonClass)); - } - - public static BaseReport fromJson(String json) { - return fromJson(json, BaseReport.class); - } - - public static <R extends BaseReport> R fromJson(String json, Class<R> jacksonClass) { - return uncheck(() -> mapper.readValue(json, jacksonClass)); - } - - /** Returns {@code this} as a {@link JsonNode}. */ - public JsonNode toJsonNode() { - return uncheck(() -> mapper.valueToTree(this)); - } - - /** Returns {@code this} as a compact JSON string. */ - public String toJson() { - return uncheck(() -> mapper.writeValueAsString(this)); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java deleted file mode 100644 index 2bc8bea013a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; - -import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author freva - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class DropDocumentsReport extends BaseReport { - private static final String REPORT_ID = "dropDocuments"; - private static final String DROPPED_AT_FIELD = "droppedAt"; - private static final String READIED_AT_FIELD = "readiedAt"; - private static final String STARTED_AT_FIELD = "startedAt"; - - private final Long droppedAt; - private final Long readiedAt; - private final Long startedAt; - - public DropDocumentsReport(@JsonProperty(CREATED_FIELD) Long createdMillisOrNull, - @JsonProperty(DROPPED_AT_FIELD) Long droppedAtOrNull, - @JsonProperty(READIED_AT_FIELD) Long readiedAtOrNull, - @JsonProperty(STARTED_AT_FIELD) Long startedAtOrNull) { - super(createdMillisOrNull, null); - this.droppedAt = droppedAtOrNull; - this.readiedAt = readiedAtOrNull; - this.startedAt = startedAtOrNull; - } - - @JsonGetter(DROPPED_AT_FIELD) - public Long droppedAt() { return droppedAt; } - - @JsonGetter(READIED_AT_FIELD) - public Long readiedAt() { return readiedAt; } - - @JsonGetter(STARTED_AT_FIELD) - public Long startedAt() { return startedAt; } - - public DropDocumentsReport withDroppedAt(long droppedAt) { - return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt); - } - - public DropDocumentsReport withStartedAt(long startedAt) { - return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt); - } - - public static String reportId() { - return REPORT_ID; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/package-info.java deleted file mode 100644 index cd8a1383966..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java deleted file mode 100644 index f16f2ca9be3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; - -import java.util.List; - -/** - * Abstraction for communicating with Orchestrator. - * - * @author bakksjo - */ -public interface Orchestrator { - - /** - * Suspends a host. - * - * @throws OrchestratorException if suspend was denied - * @throws OrchestratorNotFoundException if host is unknown to the orchestrator - */ - void suspend(String hostName); - - /** - * Resumes a host. - * - * @throws OrchestratorException if resume was denied - * @throws OrchestratorNotFoundException if host is unknown to the orchestrator - */ - void resume(String hostName); - - /** - * Suspends a list of nodes on a parent. - * - * @throws OrchestratorException if batch suspend was denied - */ - void suspend(String parentHostName, List<String> hostNames); - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java deleted file mode 100644 index 5c5c1183ea6..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; - -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; - -@SuppressWarnings("serial") -public class OrchestratorException extends ConvergenceException { - /** Creates a transient convergence exception. */ - public OrchestratorException(String message) { - this(message, false); - } - - protected OrchestratorException(String message, boolean isError) { - super(message, null, isError); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java deleted file mode 100644 index 614a79719ca..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; - -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; -import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; -import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; -import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; - -import java.net.URI; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * @author stiankri - * @author bakksjo - * @author dybis - */ -public class OrchestratorImpl implements Orchestrator { - private static final Logger logger = Logger.getLogger(OrchestratorImpl.class.getName()); - - // The server-side Orchestrator has an internal timeout of 10s. - // - // Note: A 409 has been observed to be returned after 33s in a case possibly involving - // zk leader election (which is unfortunate as it is difficult to differentiate between - // transient timeouts (do not allow suspend on timeout) and the config server being - // permanently down (allow suspend)). For now we'd like to investigate such long - // requests so keep the timeout low(er). - private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(15); - - // TODO: Find a way to avoid duplicating this (present in orchestrator's services.xml also). - private static final String ORCHESTRATOR_PATH_PREFIX = "/orchestrator"; - static final String ORCHESTRATOR_PATH_PREFIX_HOST_API - = ORCHESTRATOR_PATH_PREFIX + "/v1/hosts"; - static final String ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API - = ORCHESTRATOR_PATH_PREFIX + "/v1/suspensions/hosts"; - - private final ConfigServerApi configServerApi; - - public OrchestratorImpl(ConfigServerApi configServerApi) { - this.configServerApi = configServerApi; - } - - @Override - public void suspend(final String hostName) { - UpdateHostResponse response; - try { - var params = new ConfigServerApi - .Params<UpdateHostResponse>() - .setConnectionTimeout(CONNECTION_TIMEOUT) - .setRetryPolicy(createRetryPolicyForSuspend()); - response = configServerApi.put(getSuspendPath(hostName), Optional.empty(), UpdateHostResponse.class, params); - } catch (HttpException.NotFoundException n) { - throw new OrchestratorNotFoundException("Failed to suspend " + hostName + ", host not found"); - } catch (HttpException e) { - throw new OrchestratorException("Failed to suspend " + hostName + ": " + e); - } catch (ConnectionException e) { - throw ConvergenceException.ofTransient("Failed to suspend " + hostName + ": " + e.getMessage()); - } catch (RuntimeException e) { - throw new RuntimeException("Got error on suspend", e); - } - - Optional.ofNullable(response.reason()).ifPresent(reason -> { - throw new OrchestratorException(reason.message()); - }); - } - - private static ConfigServerApi.RetryPolicy<UpdateHostResponse> createRetryPolicyForSuspend() { - return new ConfigServerApi.RetryPolicy<>() { - @Override - public boolean tryNextConfigServer(URI configServerEndpoint, UpdateHostResponse response) { - HostStateChangeDenialReason reason = response.reason(); - if (reason == null) { - return false; - } - - // The config server has likely just bootstrapped, so try the next. - if ("unknown-service-status".equals(reason.constraintName())) { - // Warn for now and until this feature has proven to work well - logger.warning("Config server at [" + configServerEndpoint + - "] failed with transient error (will try next): " + - reason.message()); - - return true; - } - - return false; - } - }; - } - - @Override - public void suspend(String parentHostName, List<String> hostNames) { - final BatchOperationResult batchOperationResult; - try { - var params = new ConfigServerApi.Params<BatchOperationResult>().setConnectionTimeout(CONNECTION_TIMEOUT); - String hostnames = String.join("&hostname=", hostNames); - String url = String.format("%s/%s?hostname=%s", ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - parentHostName, hostnames); - batchOperationResult = configServerApi.put(url, Optional.empty(), BatchOperationResult.class, params); - } catch (HttpException e) { - throw new OrchestratorException("Failed to batch suspend for " + parentHostName + ": " + e); - } catch (ConnectionException e) { - throw ConvergenceException.ofTransient("Failed to batch suspend for " + parentHostName + ": " + e.getMessage()); - } catch (RuntimeException e) { - throw new RuntimeException("Got error on batch suspend for " + parentHostName + ", with nodes " + hostNames, e); - } - - batchOperationResult.getFailureReason().ifPresent(reason -> { - throw new OrchestratorException(reason); - }); - } - - @Override - public void resume(final String hostName) { - UpdateHostResponse response; - try { - String path = getSuspendPath(hostName); - response = configServerApi.delete(path, UpdateHostResponse.class); - } catch (HttpException.NotFoundException n) { - throw new OrchestratorNotFoundException("Failed to resume " + hostName + ", host not found"); - } catch (HttpException e) { - throw new OrchestratorException("Failed to resume " + hostName + ": " + e); - } catch (ConnectionException e) { - throw ConvergenceException.ofTransient("Failed to resume " + hostName + ": " + e.getMessage()); - } catch (RuntimeException e) { - throw new RuntimeException("Got error on resume", e); - } - - Optional.ofNullable(response.reason()).ifPresent(reason -> { - throw new OrchestratorException(reason.message()); - }); - } - - private String getSuspendPath(String hostName) { - return ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java deleted file mode 100644 index 8025eb8df93..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; - -@SuppressWarnings("serial") -public class OrchestratorNotFoundException extends OrchestratorException { - public OrchestratorNotFoundException(String message) { - super(message, true); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java deleted file mode 100644 index 6c89fbce90b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java deleted file mode 100644 index af925db8b4e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java deleted file mode 100644 index a82a82e56b0..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.state; - -/** - * The healthiness of a remote Vespa server based on REST API - * - * @author hakon - */ -public enum HealthCode { - DOWN("down"), - INITIALIZING("initializing"), - UP("up"); - - private final String code; - - HealthCode(String code) { - this.code = code; - } - - public static HealthCode fromString(String code) { - return HealthCode.valueOf(code.toUpperCase()); - } - - public String asString() { - return code; - } - - @Override - public String toString() { - return asString(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java deleted file mode 100644 index 0887637d5a1..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.state; - -/** - * The /state/v1 REST API of the config server - * - * @author hakon - */ -public interface State { - /** Issue GET on /state/v1/health */ - HealthCode getHealth(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java deleted file mode 100644 index 2471069cb4a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.state; - -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; -import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; -import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; - -/** - * @author hakon - */ -public class StateImpl implements State { - private final ConfigServerApi configServerApi; - - public StateImpl(ConfigServerApi configServerApi) { - this.configServerApi = configServerApi; - } - - @Override - public HealthCode getHealth() { - try { - HealthResponse response = configServerApi.get("/state/v1/health", HealthResponse.class); - return HealthCode.fromString(response.status.code); - } catch (ConnectionException | HttpException e) { - return HealthCode.DOWN; - } - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java deleted file mode 100644 index d0b94324941..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.state.bindings; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Response from /state/v1/health - * - * @author hakon - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class HealthResponse { - @JsonProperty("status") - public Status status = new Status(); - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Status { - @JsonProperty("code") - public String code = "down"; - - @Override - public String toString() { - return "Status{" + - "code='" + code + '\'' + - '}'; - } - } - - @Override - public String toString() { - return "HealthResponse{" + - "status=" + status + - '}'; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/package-info.java deleted file mode 100644 index fd237ec6cb4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.configserver.state; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java deleted file mode 100644 index f6f9ebd79e9..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.config.provision.DockerImage; - -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * A Podman container. - * - * @author mpolden - */ -public class Container extends PartialContainer { - - private final String hostname; - private final ContainerResources resources; - private final int conmonPid; - private final List<Network> networks; - - public Container(ContainerId id, ContainerName name, Instant createdAt, State state, String imageId, DockerImage image, - Map<String, String> labels, int pid, int conmonPid, String hostname, - ContainerResources resources, List<Network> networks, boolean managed) { - super(id, name, createdAt, state, imageId, image, labels, pid, managed); - this.hostname = Objects.requireNonNull(hostname); - this.resources = Objects.requireNonNull(resources); - this.conmonPid = conmonPid; - this.networks = List.copyOf(Objects.requireNonNull(networks)); - } - - /** The hostname of this, if any */ - public String hostname() { - return hostname; - } - - /** Resource limits for this*/ - public ContainerResources resources() { - return resources; - } - - /** Pid of the conmon process for this container */ - public int conmonPid() { - return conmonPid; - } - - /** The networks used by this */ - public List<Network> networks() { - return networks; - } - - @Override - public String toString() { - return "Container{" + - "hostname='" + hostname + '\'' + - ", resources=" + resources + - ", conmonPid=" + conmonPid + - ", networks=" + networks + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - Container that = (Container) o; - return conmonPid == that.conmonPid && hostname.equals(that.hostname) && resources.equals(that.resources) && networks.equals(that.networks); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), hostname, resources, conmonPid, networks); - } - - /** The network of a container */ - public record Network(String name, String ipv4Address) { - public Network { - Objects.requireNonNull(name); - Objects.requireNonNull(ipv4Address); - } - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java deleted file mode 100644 index 26c3ba2a45b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.container.image.Image; -import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; - -import java.time.Duration; -import java.util.List; -import java.util.Optional; - -/** - * Interface for a container engine, such as Docker or Podman. - * - * @author mpolden - */ -public interface ContainerEngine { - - /** - * Create a new container - * @return ContainerData that can be used to write files inside container - */ - ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources); - - /** Start a created container */ - void startContainer(NodeAgentContext context); - - /** Update an existing container with new resources */ - void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources); - - /** Remove given container. The container will be stopped if necessary */ - void removeContainer(TaskContext context, PartialContainer container); - - /** Get container for given context */ - Optional<Container> getContainer(NodeAgentContext context); - - /** Returns all containers known by this */ - List<PartialContainer> listContainers(TaskContext context); - - /** Returns the network interface used by container in given context */ - String networkInterface(NodeAgentContext context); - - /** Execute command inside container as given user. Ignores non-zero exit code */ - CommandResult execute(NodeAgentContext context, UnixUser user, Duration timeout, String... command); - - /** Execute command inside the container's network namespace. Throws on non-zero exit code */ - CommandResult executeInNetworkNamespace(NodeAgentContext context, CommandLine.Options options, String... command); - - default CommandResult executeInNetworkNamespace(NodeAgentContext context, String... command) { - return executeInNetworkNamespace(context, new CommandLine.Options(), command); - } - - /** Download given image */ - void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials); - - /** Returns whether given image is already downloaded */ - boolean hasImage(TaskContext context, DockerImage image); - - /** Remove image by id */ - void removeImage(TaskContext context, String id); - - /** Returns images available in this */ - List<Image> listImages(TaskContext context); - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerId.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerId.java deleted file mode 100644 index 5a800efcbd0..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerId.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// -package com.yahoo.vespa.hosted.node.admin.container; - -import java.util.Objects; - -/** - * The ID of a container. - * - * @author hakon - */ -public class ContainerId { - private final String id; - - public ContainerId(String id) { - this.id = Objects.requireNonNull(id, "id cannot be null"); - } - - @Override - public String toString() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ContainerId that = (ContainerId) o; - return id.equals(that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerName.java deleted file mode 100644 index c504e38575c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerName.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import java.util.Objects; -import java.util.regex.Pattern; - -/** - * Type-safe value wrapper for docker container names. - * - * @author bakksjo - */ -public class ContainerName implements Comparable<ContainerName> { - - private static final Pattern LEGAL_CONTAINER_NAME_PATTERN = Pattern.compile("^[a-zA-Z0-9_-]+$"); - private final String name; - - public ContainerName(final String name) { - this.name = Objects.requireNonNull(name); - if (! LEGAL_CONTAINER_NAME_PATTERN.matcher(name).matches()) { - throw new IllegalArgumentException("Illegal container name: " + name + ". Must match " + - LEGAL_CONTAINER_NAME_PATTERN.pattern()); - } - } - - public String asString() { - return name; - } - - public static ContainerName fromHostname(final String hostName) { - return new ContainerName(hostName.split("\\.", 2)[0]); - } - - @Override - public int hashCode() { - return name.hashCode(); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof ContainerName other)) { - return false; - } - - return Objects.equals(name, other.name); - } - - @Override - public String toString() { - return getClass().getSimpleName() + " {" - + " name=" + name - + " }"; - } - - @Override - public int compareTo(ContainerName o) { - return name.compareTo(o.name); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java deleted file mode 100644 index a737b049e11..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import java.util.Objects; - -/** - * Container network modes supported by node-admin. - * - * @author hakon - */ -public enum ContainerNetworkMode { - - /** Network Prefix-Translated networking. */ - NPT("vespa-bridge"), - - /** A host running a single container in the host network namespace. */ - HOST_NETWORK("host"); - - private final String networkName; - - ContainerNetworkMode(String networkName) { - this.networkName = Objects.requireNonNull(networkName); - } - - public String networkName() { - return networkName; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java deleted file mode 100644 index ce26f8e69e7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImageDownloader; -import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImagePruner; -import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; - -import java.nio.file.FileSystem; -import java.time.Duration; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * High-level interface for container operations. Code managing containers should use this and not - * {@link ContainerEngine} directly. - * - * @author hakonhall - * @author mpolden - */ -public class ContainerOperations { - - private final ContainerEngine containerEngine; - private final ContainerImageDownloader imageDownloader; - private final ContainerImagePruner imagePruner; - private final ContainerStatsCollector containerStatsCollector; - - public ContainerOperations(ContainerEngine containerEngine, Cgroup cgroup, FileSystem fileSystem, Timer timer) { - this.containerEngine = Objects.requireNonNull(containerEngine); - this.imageDownloader = new ContainerImageDownloader(containerEngine, timer); - this.imagePruner = new ContainerImagePruner(containerEngine, timer); - this.containerStatsCollector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem); - } - - public ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources) { - return containerEngine.createContainer(context, containerResources); - } - - public void startContainer(NodeAgentContext context) { - containerEngine.startContainer(context); - } - - public void removeContainer(NodeAgentContext context, Container container) { - containerEngine.removeContainer(context, container); - } - - public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { - containerEngine.updateContainer(context, containerId, containerResources); - } - - public Optional<Container> getContainer(NodeAgentContext context) { - return containerEngine.getContainer(context); - } - - /** Pull image asynchronously. Returns true if image is still downloading and false if download is complete */ - public boolean pullImageAsyncIfNeeded(TaskContext context, DockerImage dockerImage, RegistryCredentialsProvider credentialsProvider) { - return !imageDownloader.get(context, dockerImage, credentialsProvider); - } - - /** Executes a command inside container identified by given context. Does NOT throw on non-zero exit code */ - public CommandResult executeCommandInContainer(NodeAgentContext context, UnixUser user, String... command) { - return executeCommandInContainer(context, user, CommandLine.DEFAULT_TIMEOUT, command); - } - - /** Execute command inside container identified by given context. Does NOT throw on non-zero exit code */ - public CommandResult executeCommandInContainer(NodeAgentContext context, UnixUser user, Duration timeout, String... command) { - return containerEngine.execute(context, user, timeout, command); - } - - /** Execute command in inside containers network namespace, identified by given context. Throws on non-zero exit code */ - public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) { - return executeCommandInNetworkNamespace(context, new CommandLine.Options(), command); - } - - public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, CommandLine.Options options, String... command) { - return containerEngine.executeInNetworkNamespace(context, options, command); - } - - /** Resume node. Resuming a node means that it is ready to receive traffic */ - public String resumeNode(NodeAgentContext context) { - return executeNodeCtlInContainer(context, "resume"); - } - - /** - * Suspend node and return output. Suspending a node means the node should be taken temporarily offline, - * such that maintenance of the node can be done (upgrading, rebooting, etc). - */ - public String suspendNode(NodeAgentContext context) { - return executeNodeCtlInContainer(context, "suspend"); - } - - /** Restart Vespa inside container. Same as running suspend, stop, start and resume */ - public String restartVespa(NodeAgentContext context) { - return executeNodeCtlInContainer(context, "restart-vespa"); - } - - /** Start Vespa inside container */ - public String startServices(NodeAgentContext context) { - return executeNodeCtlInContainer(context, "start"); - } - - /** Stop Vespa inside container */ - public String stopServices(NodeAgentContext context) { - return executeNodeCtlInContainer(context, "stop"); - } - - /** Get container statistics */ - public Optional<ContainerStats> getContainerStats(NodeAgentContext context) { - String iface = containerEngine.networkInterface(context); - return getContainer(context).flatMap(container -> containerStatsCollector.collect(context, container.id(), container.pid(), iface)); - } - - /** Returns true if no containers managed by node-admin are running */ - public boolean noManagedContainersRunning(TaskContext context) { - return containerEngine.listContainers(context).stream() - .filter(PartialContainer::managed) - .noneMatch(container -> container.state() == Container.State.running); - } - - /** - * Stop and remove all managed containers except the given ones - * - * @return true if any containers were removed - */ - public boolean retainManagedContainers(TaskContext context, Set<ContainerName> containerNames) { - return containerEngine.listContainers(context).stream() - .filter(PartialContainer::managed) - .filter(container -> !containerNames.contains(container.name())) - .peek(container -> containerEngine.removeContainer(context, container)) - .count() > 0; - } - - /** Deletes the local images that are currently not in use by any container and not recently used. */ - public boolean deleteUnusedContainerImages(TaskContext context, List<DockerImage> excludes, Duration minImageAgeToDelete) { - List<String> excludedRefs = excludes.stream().map(DockerImage::asString).toList(); - return imagePruner.removeUnusedImages(context, excludedRefs, minImageAgeToDelete); - } - - private String executeNodeCtlInContainer(NodeAgentContext context, String program) { - String[] command = new String[] {context.paths().underVespaHome("bin/vespa-nodectl").pathInContainer(), program}; - return executeCommandInContainer(context, context.users().vespa(), command).getOutput(); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java deleted file mode 100644 index 05398e90053..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResources.java +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import java.util.Objects; - -/** - * @author freva - */ -public class ContainerResources { - - public static final ContainerResources UNLIMITED = ContainerResources.from(0, 0, 0); - public static final int CPU_PERIOD_US = 100_000; // 100 ms - - /** - * Hard limit on container's CPU usage: Implemented using Completely Fair Scheduler (CFS) by allocating a given - * time within a given period, Container's processes are not bound to any specific CPU, which may create significant - * performance degradation as processes are scheduled on another CPU after exhausting the quota. - */ - private final double cpus; - - /** - * Soft limit on container's CPU usage: When plenty of CPU cycles are available, all containers use as much - * CPU as they need. It prioritizes container CPU resources for the available CPU cycles. - * It does not guarantee or reserve any specific CPU access. - */ - private final int cpuShares; - - /** The maximum amount, in bytes, of memory the container can use. */ - private final long memoryBytes; - - public ContainerResources(double cpus, int cpuShares, long memoryBytes) { - this.cpus = cpus; - this.cpuShares = cpuShares; - this.memoryBytes = memoryBytes; - - if (cpus < 0) - throw new IllegalArgumentException("CPUs must be a positive number or 0 for unlimited, was " + cpus); - if (cpuShares != 0 && (cpuShares < 2 || cpuShares > 262_144)) - throw new IllegalArgumentException("CPU shares must be a positive integer in [2, 262144] or 0 for unlimited, was " + cpuShares); - if (memoryBytes < 0) - throw new IllegalArgumentException("memoryBytes must be a positive integer or 0 for unlimited, was " + memoryBytes); - } - - /** - * Create container resources from required fields. - * - * @param maxVcpu the amount of vcpu that allocation policies should allocate exclusively to this container. - * This is a hard upper limit. To allow an unlimited amount use 0. - * @param minVcpu the minimal amount of vcpu dedicated to this container. - * To avoid dedicating any cpu at all, use 0. - * @param memoryGb the amount of memory that allocation policies should allocate to this container. - * This is a hard upper limit. To allow the container to allocate an unlimited amount use 0. - * @return the container resources encapsulating the parameters - */ - public static ContainerResources from(double maxVcpu, double minVcpu, double memoryGb) { - return new ContainerResources(maxVcpu, - (int) Math.round(32 * minVcpu), - (long) ((1L << 30) * memoryGb)); - } - - public double cpus() { - return cpus; - } - - /** Returns the CFS CPU quota per {@link #cpuPeriod()}, or -1 if disabled. */ - public int cpuQuota() { - return cpus > 0 ? (int) (cpus * CPU_PERIOD_US) : -1; - } - - /** Duration (in µs) of a single period used as the basis for process scheduling */ - public int cpuPeriod() { - return CPU_PERIOD_US; - } - - public int cpuShares() { - return cpuShares; - } - - public long memoryBytes() { - return memoryBytes; - } - - /** Returns true iff the memory component(s) of between <code>this</code> and <code>other</code> are equal */ - public boolean equalsMemory(ContainerResources other) { - return memoryBytes == other.memoryBytes; - } - - /** Returns true iff the CPU component(s) of between <code>this</code> and <code>other</code> are equal */ - public boolean equalsCpu(ContainerResources other) { - return Math.abs(other.cpus - cpus) < 0.0001 && - // When using CGroups V2, CPU shares (range [2, 262144]) is mapped to CPU weight (range [1, 10000]), - // because there are ~26.2 shares/weight, we must allow for small deviation in cpuShares - // when comparing ContainerResources created from NodeResources vs one created from reading the - // CGroups weight file - Math.abs(cpuShares - other.cpuShares) < 28; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ContainerResources that = (ContainerResources) o; - return equalsMemory(that) && equalsCpu(that); - } - - @Override - public int hashCode() { - return Objects.hash(cpus, cpuShares, memoryBytes); - } - - - /** Returns only the memory component(s) of {@link #toString()} */ - public String toStringMemory() { - return (memoryBytes > 0 ? memoryBytes + "B" : "unlimited") + " memory"; - } - - /** Returns only the CPU component(s) of {@link #toString()} */ - public String toStringCpu() { - return (cpus > 0 ? String.format("%.2f", cpus) : "unlimited") +" CPUs, " + - (cpuShares > 0 ? cpuShares : "unlimited") + " CPU Shares"; - } - - @Override - public String toString() { - return toStringCpu() + ", " + toStringMemory(); - } - - public ContainerResources withMemoryBytes(long memoryBytes) { - return new ContainerResources(cpus, cpuShares, memoryBytes); - } - - public ContainerResources withUnlimitedCpus() { - return new ContainerResources(0, 0, memoryBytes); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java deleted file mode 100644 index 9c1b8db144c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStats.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import ai.vespa.validation.Validation; - -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * CPU, GPU, memory and network statistics collected from a container. - * - * @author freva - */ -public record ContainerStats(Map<String, NetworkStats> networks, - MemoryStats memoryStats, - CpuStats cpuStats, - List<GpuStats> gpuStats) { - - public ContainerStats(Map<String, NetworkStats> networks, MemoryStats memoryStats, CpuStats cpuStats, List<GpuStats> gpuStats) { - this.networks = Collections.unmodifiableMap(new LinkedHashMap<>(Objects.requireNonNull(networks))); - this.memoryStats = Objects.requireNonNull(memoryStats); - this.cpuStats = Objects.requireNonNull(cpuStats); - this.gpuStats = List.copyOf(Objects.requireNonNull(gpuStats)); - } - - /** - * Statistics for network usage - * - * @param rxBytes received bytes - * @param rxDropped received bytes, which were dropped - * @param rxErrors received errors - * @param txBytes transmitted bytes - * @param txDropped transmitted bytes, which were dropped - * @param txErrors transmission errors - */ - public record NetworkStats(long rxBytes, long rxDropped, long rxErrors, long txBytes, long txDropped, long txErrors) {} - - /** - * Statistics for memory usage - * - * @param cache memory used by cache in bytes - * @param usage memory usage in bytes - * @param limit memory limit in bytes - * @param sock network transmission buffers in bytes - * @param slab in-kernel data structures in bytes - * @param slabReclaimable part of "slab" that might be reclaimed in bytes - * @param anon anonymous mappings in bytes - */ - public record MemoryStats(long cache, long usage, long limit, long sock, long slab, long slabReclaimable, long anon) { - public MemoryStats(long cache, long usage, long limit) { this(cache, usage, limit, 0, 0, 0, 0); } - } - - /** - * Statistics for CPU usage - * - * @param onlineCpus CPU cores - * @param systemCpuUsage Total CPU time (in µs) spent executing all the processes on this host - * @param totalUsage Total CPU time (in µs) spent running all the processes in this container - * @param usageInKernelMode Total CPU time (in µs) spent in kernel mode while executing processes in this container - * @param throttledTime Total CPU time (in µs) processes in this container were throttled for - * @param throttlingActivePeriods Number of periods with throttling enabled for this container - * @param throttledPeriods Number of periods this container hit the throttling limit - */ - public record CpuStats(int onlineCpus, - long systemCpuUsage, - long totalUsage, - long usageInKernelMode, - long throttledTime, - long throttlingActivePeriods, - long throttledPeriods) {} - - /** - * GPU statistics - * - * @param deviceNumber GPU device number - * @param loadPercentage Load/utilization in % - * @param memoryTotalBytes Total memory, in bytes - * @param memoryUsedBytes Memory used, in bytes - */ - public record GpuStats(int deviceNumber, int loadPercentage, long memoryTotalBytes, long memoryUsedBytes) { - - public GpuStats { - Validation.requireAtLeast(deviceNumber, "deviceNumber", 0); - Validation.requireAtLeast(loadPercentage, "loadPercentage", 0); - Validation.requireAtLeast(memoryTotalBytes, "memoryTotalBytes", 0L); - Validation.requireAtLeast(memoryUsedBytes, "memoryUsedBytes", 0L); - } - - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java deleted file mode 100644 index aa6f8d8f5f6..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; -import com.yahoo.vespa.hosted.node.admin.cgroup.CpuController; -import com.yahoo.vespa.hosted.node.admin.cgroup.Size; -import com.yahoo.vespa.hosted.node.admin.cgroup.MemoryController; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Stream; - -/** - * Collects CPU, GPU, memory and network statistics for a container. - * - * Uses same approach as runc: https://github.com/opencontainers/runc/tree/master/libcontainer/cgroups/fs - * - * @author mpolden - */ -class ContainerStatsCollector { - - private final ContainerEngine containerEngine; - private final FileSystem fileSystem; - private final Cgroup rootCgroup; - private final int onlineCpus; - - ContainerStatsCollector(ContainerEngine containerEngine, Cgroup rootCgroup, FileSystem fileSystem) { - this(containerEngine, rootCgroup, fileSystem, Runtime.getRuntime().availableProcessors()); - } - - ContainerStatsCollector(ContainerEngine containerEngine, Cgroup rootCgroup, FileSystem fileSystem, int onlineCpus) { - this.containerEngine = Objects.requireNonNull(containerEngine); - this.fileSystem = Objects.requireNonNull(fileSystem); - this.rootCgroup = Objects.requireNonNull(rootCgroup); - this.onlineCpus = onlineCpus; - } - - /** Collect statistics for given container ID and PID */ - public Optional<ContainerStats> collect(NodeAgentContext context, ContainerId containerId, int pid, String iface) { - try { - ContainerStats.CpuStats cpuStats = collectCpuStats(containerId); - ContainerStats.MemoryStats memoryStats = collectMemoryStats(containerId); - Map<String, ContainerStats.NetworkStats> networkStats = Map.of(iface, collectNetworkStats(iface, pid)); - List<ContainerStats.GpuStats> gpuStats = collectGpuStats(context); - return Optional.of(new ContainerStats(networkStats, memoryStats, cpuStats, gpuStats)); - } catch (NoSuchFileException ignored) { - return Optional.empty(); // Container disappeared while we collected stats - } catch (UncheckedIOException e) { - if (e.getCause() != null && e.getCause() instanceof NoSuchFileException) - return Optional.empty(); - throw e; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private List<ContainerStats.GpuStats> collectGpuStats(NodeAgentContext context) { - boolean hasGpu = Files.exists(fileSystem.getPath("/dev/nvidia0")); - if (!hasGpu) { - return List.of(); - } - Stream<String> lines = containerEngine.execute(context, UnixUser.ROOT, Duration.ofSeconds(5), - "nvidia-smi", - "--query-gpu=index,utilization.gpu,memory.total,memory.free", - "--format=csv,noheader,nounits") - .getOutputLinesStream(); - return lines.map(ContainerStatsCollector::parseGpuStats).toList(); - } - - private static ContainerStats.GpuStats parseGpuStats(String s) { - String[] fields = fields(s, ",\\s*"); - if (fields.length < 4) throw new IllegalArgumentException("Could not parse GPU stats from '" + s + "'"); - int deviceNumber = Integer.parseInt(fields[0]); - int loadPercentage = Integer.parseInt(fields[1]); - long mega = 2 << 19; - long memoryTotalBytes = Long.parseLong(fields[2]) * mega; - long memoryFreeBytes = Long.parseLong(fields[3]) * mega; - long memoryUsedBytes = memoryTotalBytes - memoryFreeBytes; - return new ContainerStats.GpuStats(deviceNumber, loadPercentage, memoryTotalBytes, memoryUsedBytes); - } - - private ContainerStats.CpuStats collectCpuStats(ContainerId containerId) throws IOException { - Map<CpuController.StatField, Long> cpuStats = rootCgroup.resolveContainer(containerId).cpu().readStats(); - return new ContainerStats.CpuStats(onlineCpus, - systemCpuUsage(), - cpuStats.get(CpuController.StatField.TOTAL_USAGE_USEC), - cpuStats.get(CpuController.StatField.SYSTEM_USAGE_USEC), - cpuStats.get(CpuController.StatField.THROTTLED_TIME_USEC), - cpuStats.get(CpuController.StatField.TOTAL_PERIODS), - cpuStats.get(CpuController.StatField.THROTTLED_PERIODS)); - } - - private ContainerStats.MemoryStats collectMemoryStats(ContainerId containerId) throws IOException { - MemoryController memoryController = rootCgroup.resolveContainer(containerId).memory(); - Size max = memoryController.readMax(); - long memoryUsageInBytes = memoryController.readCurrent().value(); - var stats = memoryController.readStat(); - return new ContainerStats.MemoryStats( - stats.file().value(), memoryUsageInBytes, max.isMax() ? -1 : max.value(), - stats.sock().value(), stats.slab().value(), stats.slabReclaimable().value(), stats.anon().value()); - } - - private ContainerStats.NetworkStats collectNetworkStats(String iface, int containerPid) throws IOException { - for (var line : Files.readAllLines(netDevPath(containerPid))) { - String[] fields = fields(line.trim()); - if (fields.length < 17 || !fields[0].equals(iface + ":")) continue; - - long rxBytes = Long.parseLong(fields[1]); - long rxErrors = Long.parseLong(fields[3]); - long rxDropped = Long.parseLong(fields[4]); - - long txBytes = Long.parseLong(fields[9]); - long txErrors = Long.parseLong(fields[11]); - long txDropped = Long.parseLong(fields[12]); - - return new ContainerStats.NetworkStats(rxBytes, rxDropped, rxErrors, txBytes, txDropped, txErrors); - } - throw new IllegalArgumentException("No statistics found for interface " + iface); - } - - /** Returns total CPU time in µs spent executing all the processes on this host */ - private long systemCpuUsage() throws IOException { - long ticks = parseLong(Files.readAllLines(fileSystem.getPath("/proc/stat")), "cpu"); - return userHzToMicroSeconds(ticks); - } - - private long parseLong(List<String> lines, String fieldName) { - long value = 0; - for (var line : lines) { - String[] fields = fields(line); - if (fields.length < 2 || !fields[0].equals(fieldName)) continue; - for (int i = 1; i < fields.length; i++) { - value += Long.parseLong(fields[i]); - } - break; - } - return value; - } - - private Path netDevPath(int containerPid) { - return fileSystem.getPath("/proc/" + containerPid + "/net/dev"); - } - - static long userHzToMicroSeconds(long ticks) { - // Ideally we would read this from _SC_CLK_TCK, but then we need JNI. However, in practice this is always 100 on x86 Linux - return ticks * 10_000; - } - - private static String[] fields(String s) { - return fields(s, "\\s+"); - } - - private static String[] fields(String s, String regex) { - return s.trim().split(regex); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java deleted file mode 100644 index c9310897df9..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.config.provision.DockerImage; - -import java.time.Instant; -import java.util.Map; -import java.util.Objects; - -/** - * A partial container, containing only fields returned by a container list command such as 'podman ps'. - * - * @author mpolden - */ -public class PartialContainer { - - private final ContainerId id; - private final ContainerName name; - private final Instant createdAt; - private final State state; - private final String imageId; - private final DockerImage image; - private final Map<String, String> labels; - private final int pid; - private final boolean managed; - - public PartialContainer(ContainerId id, ContainerName name, Instant createdAt, State state, String imageId, - DockerImage image, Map<String, String> labels, int pid, boolean managed) { - this.id = Objects.requireNonNull(id); - this.name = Objects.requireNonNull(name); - this.createdAt = Objects.requireNonNull(createdAt); - this.state = Objects.requireNonNull(state); - this.imageId = Objects.requireNonNull(imageId); - this.image = Objects.requireNonNull(image); - this.labels = Map.copyOf(Objects.requireNonNull(labels)); - this.pid = pid; - this.managed = managed; - } - - /** A unique identifier for this. Typically generated by the container engine */ - public ContainerId id() { - return id; - } - - /** The given name of this */ - public ContainerName name() { - return name; - } - - /** Timestamp when this container was created */ - public Instant createdAt() { - return createdAt; - } - - /** Current state of this */ - public State state() { - return state; - } - - /** A unique identifier for the image in use by this */ - public String imageId() { - return imageId; - } - - /** The image in use by this */ - public DockerImage image() { - return image; - } - - /** The labels set on this */ - public Map<String, String> labels() { - return labels; - } - - /** The PID of this */ - public int pid() { - return pid; - } - - /** Returns whether this container is managed by node-admin */ - public boolean managed() { - return managed; - } - - /** Returns the value of given label key */ - public String label(String key) { - String labelValue = labels.get(key); - if (labelValue == null) throw new IllegalArgumentException("No such label '" + key + "'"); - return labelValue; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PartialContainer that = (PartialContainer) o; - return pid == that.pid && managed == that.managed && id.equals(that.id) && name.equals(that.name) && createdAt.equals(that.createdAt) && state == that.state && imageId.equals(that.imageId) && image.equals(that.image) && labels.equals(that.labels); - } - - @Override - public int hashCode() { - return Objects.hash(id, name, createdAt, state, imageId, image, labels, pid, managed); - } - - /** The state of a container */ - public enum State { - - unknown, - configured, - created, - running, - stopped, - paused, - exited, - removing, - stopping; - - public boolean isRunning() { - return this == running; - } - - public static Container.State from(String state) { - switch (state) { - case "unknown": return unknown; - case "configured": return configured; - case "created": return created; - case "running": return running; - case "stopped": return stopped; - case "paused": return paused; - case "exited": return exited; - case "removing": return removing; - case "stopping": return stopping; - } - throw new IllegalArgumentException("Invalid state '" + state + "'"); - } - - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentials.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentials.java deleted file mode 100644 index 7a5f46dab74..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentials.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import java.util.Objects; - -/** - * Credentials for a container registry server. - * - * @author mpolden - */ -public record RegistryCredentials(String username, String password) { - - public static final RegistryCredentials none = new RegistryCredentials("", ""); - - public RegistryCredentials { - Objects.requireNonNull(username); - Objects.requireNonNull(password); - } - - @Override - public String toString() { - return "registry credentials [username=" + username + ",password=<hidden>]"; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentialsProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentialsProvider.java deleted file mode 100644 index 8711227058a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/RegistryCredentialsProvider.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -/** - * Interface for retrieving credentials for a container registry. - * - * @author mpolden - */ -public interface RegistryCredentialsProvider { - - RegistryCredentials get(); - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java deleted file mode 100644 index ab2adc061fd..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.image; - -import com.yahoo.concurrent.DaemonThreadFactory; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.container.ContainerEngine; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentialsProvider; - -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Download a container image asynchronously. - * - * @author mpolden - */ -public class ContainerImageDownloader { - - private static final Logger LOG = Logger.getLogger(ContainerImageDownloader.class.getName()); - - private final ContainerEngine containerEngine; - private final Timer timer; - - private final ExecutorService executorService = Executors.newSingleThreadExecutor( - new DaemonThreadFactory("container-image-downloader")); // Download one image at a time - private final Set<DockerImage> pendingDownloads = Collections.synchronizedSet(new HashSet<>()); - - public ContainerImageDownloader(ContainerEngine containerEngine, Timer timer) { - this.containerEngine = Objects.requireNonNull(containerEngine); - this.timer = Objects.requireNonNull(timer); - } - - /** - * Download given container image. - * - * @return true if the image download has completed. - */ - public boolean get(TaskContext context, DockerImage image, RegistryCredentialsProvider credentialsProvider) { - if (pendingDownloads.contains(image)) return false; - if (containerEngine.hasImage(context, image)) return true; - executorService.submit(() -> { - try { - Instant start = timer.currentTime(); - containerEngine.pullImage(context, image, credentialsProvider.get()); - LOG.log(Level.INFO, "Downloaded container image " + image + " in " + Duration.between(start, timer.currentTime())); - } catch (RuntimeException e) { - LOG.log(Level.SEVERE, "Failed to download container image " + image, e); - } finally { - pendingDownloads.remove(image); - } - }); - pendingDownloads.add(image); - return false; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java deleted file mode 100644 index 51bf238fa67..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.image; - -import com.yahoo.collections.Pair; -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.container.ContainerEngine; -import com.yahoo.vespa.hosted.node.admin.container.PartialContainer; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * This class removes container images that have not been recently used by any containers. - * - * <p>Definitions: - * <ul> - * <li>Every image has exactly 1 id</li> - * <li>Every image has between 0..n tags, see - * <a href="https://docs.docker.com/engine/reference/commandline/tag/">docker tag</a> for more</li> - * <li>Every image has 0..1 parent ids</li> - * </ul> - * - * <p>Limitations: - * <ol> - * <li>Image that has more than 1 tag cannot be deleted by ID</li> - * <li>Deleting a tag of an image with multiple tags will only remove the tag, the image with the - * remaining tags will remain</li> - * <li>Deleting the last tag of an image will delete the entire image.</li> - * <li>Image cannot be deleted if: - * <p>- It has 1 or more children - * <p>- A container uses it - * </li> - * </ol> - * - * @author freva - * @author mpolden - */ -public class ContainerImagePruner { - - private static final Logger LOG = Logger.getLogger(ContainerImagePruner.class.getName()); - - private final Timer timer; - private final ContainerEngine containerEngine; - - private final Map<String, Instant> lastTimeUsedByImageId = new ConcurrentHashMap<>(); - - public ContainerImagePruner(ContainerEngine containerEngine, Timer timer) { - this.containerEngine = Objects.requireNonNull(containerEngine); - this.timer = Objects.requireNonNull(timer); - } - - /** - * Remove unused container images. - * - * Note: This method must be called frequently enough to see all containers to know which images are being used. - * - * @param excludedRefs List of image references (tag or id) to keep, regardless of their status - * @param minAge Minimum age of for image to be removed - * @return true if any image was remove - */ - public boolean removeUnusedImages(TaskContext context, List<String> excludedRefs, Duration minAge) { - List<Image> images = containerEngine.listImages(context); - List<PartialContainer> containers = containerEngine.listContainers(context); - - Map<String, Image> imageByImageId = images.stream().collect(Collectors.toMap(Image::id, Function.identity())); - - // The set of images that we want to keep is: - // 1. The images that were recently used - // 2. The images that were explicitly excluded - Set<String> imagesToKeep = Stream - .concat( - updateRecentlyUsedImageIds(images, containers, minAge).stream(), // 1 - referencesToImages(excludedRefs, images).stream()) // 2 - .collect(Collectors.toSet()); - - // Now take all the images we have locally - List<Image> imagesToRemove = imageByImageId.keySet().stream() - // filter out images we want to keep - .filter(imageId -> !imagesToKeep.contains(imageId)) - .map(imageByImageId::get) - .collect(Collectors.toCollection(ArrayList::new)); - - // We cannot delete an image that is referenced by other images as parent. Computing parent image is complicated, see - // https://github.com/containers/podman/blob/d7b2f03f8a5d0e3789ac185ea03989463168fb76/vendor/github.com/containers/common/libimage/layer_tree.go#L235:L299 - // https://github.com/containers/podman/blob/d7b2f03f8a5d0e3789ac185ea03989463168fb76/vendor/github.com/containers/common/libimage/oci.go#L30:L97 - // In practice, our images do not have any parents on prod machines, so we should be able to delete in any - // order. In case we ever do get a parent on a host somehow, we could get stuck if we always attempt to delete - // in wrong order, so shuffle first to ensure this eventually converges - Collections.shuffle(imagesToRemove); - - imagesToRemove.forEach(image -> { - // Deleting an image by image ID with multiple tags will fail -> delete by tags instead - referencesOf(image).forEach(imageReference -> { - LOG.info("Deleting unused image " + imageReference); - containerEngine.removeImage(context, imageReference); - }); - lastTimeUsedByImageId.remove(image.id()); - }); - return !imagesToRemove.isEmpty(); - } - - private Set<String> updateRecentlyUsedImageIds(List<Image> images, List<PartialContainer> containers, Duration minImageAgeToDelete) { - final Instant now = timer.currentTime(); - - // Add any already downloaded image to the list once - images.forEach(image -> lastTimeUsedByImageId.putIfAbsent(image.id(), now)); - - // Update last used time for all current containers - containers.forEach(container -> lastTimeUsedByImageId.put(container.imageId(), now)); - - // Return list of images that have been used within minImageAgeToDelete - return lastTimeUsedByImageId.entrySet().stream() - .filter(entry -> Duration.between(entry.getValue(), now).minus(minImageAgeToDelete).isNegative()) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()); - } - - /** - * Map given references (image tags or ids) to images. - * - * This only works if the given tag is actually present locally. This is fine, because if it isn't - we can't delete - * it, so no harm done. - */ - private Set<String> referencesToImages(List<String> references, List<Image> images) { - Map<String, String> imageIdByImageTag = images.stream() - .flatMap(image -> referencesOf(image).stream() - .map(repoTag -> new Pair<>(repoTag, image.id()))) - .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); - - return references.stream() - .map(ref -> imageIdByImageTag.getOrDefault(ref, ref)) - .collect(Collectors.toUnmodifiableSet()); - } - - /** - * Returns list of references to given image, preferring image tag(s), if any exist. - * - * If image is untagged, its ID is returned instead. - */ - private static List<String> referencesOf(Image image) { - if (image.names().isEmpty()) { - return List.of(image.id()); - } - return image.names().stream() - .map(tag -> { - if ("<none>:<none>".equals(tag)) return image.id(); - return tag; - }) - .toList(); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java deleted file mode 100644 index 223304f058e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.image; - -import java.util.List; -import java.util.Objects; - -/** - * This represents a container image that exists locally. - * - * @author mpolden - */ -public class Image { - - private final String id; - private final List<String> names; - - public Image(String id, List<String> names) { - this.id = Objects.requireNonNull(id); - this.names = List.copyOf(Objects.requireNonNull(names)); - } - - /** The identifier of this image */ - public String id() { - return id; - } - - /** Names for this image, such as tags or digests */ - public List<String> names() { - return names; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Image image = (Image) o; - return id.equals(image.id) && names.equals(image.names); - } - - @Override - public int hashCode() { - return Objects.hash(id, names); - } - - @Override - public String toString() { - return "image " + id; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/package-info.java deleted file mode 100644 index fa348209520..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author mpolden - */ -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.container.image; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Counter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Counter.java deleted file mode 100644 index e6d05e04965..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Counter.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -/** - * @author freva - */ -public class Counter implements MetricValue { - private final Object lock = new Object(); - - private long value = 0; - - public void increment() { - add(1L); - } - - public void add(long n) { - synchronized (lock) { - value += n; - } - } - - @Override - public Number getValue() { - synchronized (lock) { - return value; - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/DimensionMetrics.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/DimensionMetrics.java deleted file mode 100644 index 724432431cd..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/DimensionMetrics.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @author freva - */ -public class DimensionMetrics { - - private final String application; - private final Dimensions dimensions; - private final Map<String, Number> metrics; - - DimensionMetrics(String application, Dimensions dimensions, Map<String, Number> metrics) { - this.application = Objects.requireNonNull(application); - this.dimensions = Objects.requireNonNull(dimensions); - this.metrics = metrics.entrySet().stream() - .filter(DimensionMetrics::metricIsFinite) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - public String getApplication() { - return application; - } - - public Dimensions getDimensions() { - return dimensions; - } - - public Map<String, Number> getMetrics() { - return metrics; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DimensionMetrics that = (DimensionMetrics) o; - return application.equals(that.application) && - dimensions.equals(that.dimensions) && - metrics.equals(that.metrics); - } - - @Override - public int hashCode() { - return Objects.hash(application, dimensions, metrics); - } - - private static boolean metricIsFinite(Map.Entry<String, Number> metric) { - return ! (metric.getValue() instanceof Double) || Double.isFinite((double) metric.getValue()); - } - - public static class Builder { - private final String application; - private final Dimensions dimensions; - private final Map<String, Number> metrics = new HashMap<>(); - - public Builder(String application, Dimensions dimensions) { - this.application = application; - this.dimensions = dimensions; - } - - public Builder withMetric(String metricName, Number metricValue) { - metrics.put(metricName, metricValue); - return this; - } - - public DimensionMetrics build() { - return new DimensionMetrics(application, dimensions, metrics); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Dimensions.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Dimensions.java deleted file mode 100644 index 0f9144b9ca1..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Dimensions.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author freva - */ -public record Dimensions(Map<String, String> dimensionsMap) { - - public static final Dimensions NONE = new Dimensions(Map.of()); - - public Dimensions(Map<String, String> dimensionsMap) { - this.dimensionsMap = Map.copyOf(dimensionsMap); - } - - public static class Builder { - private final Map<String, String> dimensionsMap = new HashMap<>(); - - public Dimensions.Builder add(String dimensionName, String dimensionValue) { - dimensionsMap.put(dimensionName, dimensionValue); - return this; - } - - public Dimensions build() { - return new Dimensions(dimensionsMap); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Gauge.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Gauge.java deleted file mode 100644 index d97db8f0242..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Gauge.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -/** - * @author freva - */ -public class Gauge implements MetricValue { - private final Object lock = new Object(); - - private double value; - - public void sample(double x) { - synchronized (lock) { - this.value = x; - } - } - - @Override - public Number getValue() { - synchronized (lock) { - return value; - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricValue.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricValue.java deleted file mode 100644 index da05464e0be..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricValue.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -/** - * @author freva - */ -public interface MetricValue { - Number getValue(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java deleted file mode 100644 index e144f3a91e3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -import com.yahoo.component.annotation.Inject; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * Stores the latest metric for the given application, name, dimension triplet in memory - * - * @author freva - */ -public class Metrics { - // Application names used - public static final String APPLICATION_HOST = "vespa.host"; - public static final String APPLICATION_NODE = "vespa.node"; - - private final Object monitor = new Object(); - private final Map<DimensionType, Map<String, ApplicationMetrics>> metrics = new HashMap<>(); - - @Inject - public Metrics() { } - - /** - * Creates a counter metric under vespa.host application, with no dimensions and default dimension type - * See {@link #declareCounter(String, String, Dimensions, DimensionType)} - */ - public Counter declareCounter(String name) { - return declareCounter(name, Dimensions.NONE); - } - - /** - * Creates a counter metric under vespa.host application, with the given dimensions and default dimension type - * See {@link #declareCounter(String, String, Dimensions, DimensionType)} - */ - public Counter declareCounter(String name, Dimensions dimensions) { - return declareCounter(APPLICATION_HOST, name, dimensions, DimensionType.DEFAULT); - } - - /** Creates a counter metric. This method is idempotent. */ - public Counter declareCounter(String application, String name, Dimensions dimensions, DimensionType type) { - synchronized (monitor) { - return (Counter) getOrCreateApplicationMetrics(application, type) - .computeIfAbsent(dimensions, d -> new HashMap<>()) - .computeIfAbsent(name, n -> new Counter()); - } - } - - /** - * Creates a gauge metric under vespa.host application, with no dimensions and default dimension type - * See {@link #declareGauge(String, String, Dimensions, DimensionType)} - */ - public Gauge declareGauge(String name) { - return declareGauge(name, Dimensions.NONE); - } - - /** - * Creates a gauge metric under vespa.host application, with the given dimensions and default dimension type - * See {@link #declareGauge(String, String, Dimensions, DimensionType)} - */ - public Gauge declareGauge(String name, Dimensions dimensions) { - return declareGauge(APPLICATION_HOST, name, dimensions, DimensionType.DEFAULT); - } - - /** Creates a gauge metric. This method is idempotent */ - public Gauge declareGauge(String application, String name, Dimensions dimensions, DimensionType type) { - synchronized (monitor) { - return (Gauge) getOrCreateApplicationMetrics(application, type) - .computeIfAbsent(dimensions, d -> new HashMap<>()) - .computeIfAbsent(name, n -> new Gauge()); - } - } - - public List<DimensionMetrics> getDefaultMetrics() { - return getMetricsByType(DimensionType.DEFAULT); - } - - public List<DimensionMetrics> getMetricsByType(DimensionType type) { - synchronized (monitor) { - List<DimensionMetrics> dimensionMetrics = new ArrayList<>(); - metrics.getOrDefault(type, Map.of()) - .forEach((application, applicationMetrics) -> applicationMetrics.metricsByDimensions().entrySet().stream() - .map(entry -> new DimensionMetrics(application, entry.getKey(), - entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, value -> value.getValue().getValue())))) - .forEach(dimensionMetrics::add)); - return dimensionMetrics; - } - } - - public void deleteMetricByDimension(String name, Dimensions dimensionsToRemove, DimensionType type) { - synchronized (monitor) { - Optional.ofNullable(metrics.get(type)) - .map(m -> m.get(name)) - .map(ApplicationMetrics::metricsByDimensions) - .ifPresent(m -> m.remove(dimensionsToRemove)); - } - } - - public void deleteMetricByName(String application, String metricName, DimensionType type) { - synchronized (monitor) { - Optional.ofNullable(metrics.get(type)) - .map(m -> m.get(application)) - .map(ApplicationMetrics::metricsByDimensions) - .ifPresent(dims -> - dims.values().forEach(metrics -> metrics.remove(metricName)) - ); - } - } - - Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application, DimensionType type) { - return metrics.computeIfAbsent(type, m -> new HashMap<>()) - .computeIfAbsent(application, app -> new ApplicationMetrics()) - .metricsByDimensions(); - } - - // "Application" is the monitoring application, not Vespa application - private static class ApplicationMetrics { - private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new LinkedHashMap<>(); - - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions() { - return metricsByDimensions; - } - } - - // Used to distinguish whether metrics have been populated with all tag values - public enum DimensionType { - /** Default metrics get added default dimensions set in check config */ - DEFAULT, - - /** Pretagged metrics will only get the dimensions explicitly set when creating the counter/gauge */ - PRETAGGED - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/package-info.java deleted file mode 100644 index e6ddfa2f4c8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/package-info.java deleted file mode 100644 index 86f3c31ff39..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java deleted file mode 100644 index 332a225bda3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import com.yahoo.vespa.hosted.node.admin.container.ContainerId; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; - -/** - * Wireguard task for containers. - * - * @author gjoranv - */ -public interface ContainerWireguardTask { - - void converge(NodeAgentContext context, ContainerId containerId); - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java deleted file mode 100644 index 8bfb3f86aa7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeType; -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.container.Container; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; -import com.yahoo.vespa.hosted.node.admin.maintenance.disk.CoredumpCleanupRule; -import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; -import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule; -import com.yahoo.vespa.hosted.node.admin.maintenance.disk.LinearCleanupRule; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask; -import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; - -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; - -/** - * @author freva - */ -public class StorageMaintainer { - private static final Logger logger = Logger.getLogger(StorageMaintainer.class.getName()); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter - .ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC); - - private final Terminal terminal; - private final CoredumpHandler coredumpHandler; - private final DiskCleanup diskCleanup; - private final SyncClient syncClient; - private final Timer timer; - private final Path archiveContainerStoragePath; - - // We cache disk usage to avoid doing expensive disk operations so often - private final Cache<ContainerName, DiskSize> diskUsage = CacheBuilder.newBuilder() - .maximumSize(100) - .expireAfterWrite(5, TimeUnit.MINUTES) - .build(); - - public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, DiskCleanup diskCleanup, - SyncClient syncClient, Timer timer, Path archiveContainerStoragePath) { - this.terminal = terminal; - this.coredumpHandler = coredumpHandler; - this.diskCleanup = diskCleanup; - this.syncClient = syncClient; - this.timer = timer; - this.archiveContainerStoragePath = archiveContainerStoragePath; - } - - public boolean syncLogs(NodeAgentContext context, boolean throttle) { - Optional<URI> archiveUri = context.node().archiveUri(); - if (archiveUri.isEmpty()) return false; - ApplicationId owner = context.node().owner().orElseThrow(); - - List<SyncFileInfo> syncFileInfos = FileFinder.files(context.paths().underVespaHome("logs/vespa")) - .maxDepth(2) - .stream() - .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime)) - .flatMap(fa -> SyncFileInfo.forLogFile(archiveUri.get(), fa.path(), throttle, owner).stream()) - .toList(); - - return syncClient.sync(context, syncFileInfos, throttle ? 1 : 100); - } - - public Optional<DiskSize> diskUsageFor(NodeAgentContext context) { - try { - DiskSize cachedDiskUsage = diskUsage.getIfPresent(context.containerName()); - if (cachedDiskUsage != null) return Optional.of(cachedDiskUsage); - - DiskSize diskUsageBytes = getDiskUsed(context, context.paths().of("/").pathOnHost()); - diskUsage.put(context.containerName(), diskUsageBytes); - return Optional.of(diskUsageBytes); - } catch (Exception e) { - context.log(logger, Level.WARNING, "Failed to get disk usage", e); - return Optional.empty(); - } - } - - DiskSize getDiskUsed(TaskContext context, Path pathOnHost) { - if (!Files.exists(pathOnHost)) return DiskSize.ZERO; - - String output = terminal.newCommandLine(context) - .add("du", "-xsk", pathOnHost.toString()) - .setTimeout(Duration.ofSeconds(60)) - .executeSilently() - .getOutput(); - - String[] results = output.split("\t"); - if (results.length != 2) { - throw ConvergenceException.ofError("Result from disk usage command not as expected: " + output); - } - - return DiskSize.of(Long.parseLong(results[0]), DiskSize.Unit.kiB); - } - - public boolean cleanDiskIfFull(NodeAgentContext context) { - if (context.isDisabled(NodeAgentTask.DiskCleanup)) return false; - - double totalBytes = context.node().diskSize().bytes(); - // Delete enough bytes to get below 70% disk usage, but only if we are already using more than 80% disk - long bytesToRemove = diskUsageFor(context) - .map(diskUsage -> (long) (diskUsage.bytes() - 0.7 * totalBytes)) - .filter(bytes -> bytes > totalBytes * 0.1) - .orElse(0L); - - if (bytesToRemove > 0 && diskCleanup.cleanup(context, createCleanupRules(context), bytesToRemove)) { - diskUsage.invalidate(context.containerName()); - return true; - } - return false; - } - - private List<DiskCleanupRule> createCleanupRules(NodeAgentContext context) { - Instant start = timer.currentTime(); - double oneMonthSeconds = Duration.ofDays(30).getSeconds(); - Function<Instant, Double> monthNormalizer = instant -> Duration.between(instant, start).getSeconds() / oneMonthSeconds; - List<DiskCleanupRule> rules = new ArrayList<>(); - - rules.add(CoredumpCleanupRule.forContainer(context.paths().underVespaHome("var/crash"))); - - rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("var/tmp")).list(), - fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST)); - - if (context.node().membership().map(m -> m.type().hasContainer()).orElse(false)) { - rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("logs/vespa/access")).list(), - fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST)); - } - if (context.nodeType() == NodeType.tenant && context.node().membership().map(m -> m.type().isAdmin()).orElse(false)) - rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("logs/vespa/logarchive")).list(), - fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST)); - - if (context.nodeType() == NodeType.proxy) - rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("logs/nginx")).list(), - fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.MEDIUM)); - - return rules; - } - - /** Checks if container has any new coredumps, reports and archives them if so */ - public void handleCoreDumpsForContainer(NodeAgentContext context, Optional<Container> container, boolean throwIfCoreBeingWritten) { - if (context.isDisabled(NodeAgentTask.CoreDumps)) return; - coredumpHandler.converge(context, container.map(Container::image), throwIfCoreBeingWritten); - } - - /** - * Prepares the container-storage for the next container by deleting/archiving all the data of the current container. - * Removes old files, reports coredumps and archives container data, runs when container enters state "dirty" - */ - public void archiveNodeStorage(NodeAgentContext context) { - ContainerPath logsDirInContainer = context.paths().underVespaHome("logs"); - Path containerLogsInArchiveDir = archiveContainerStoragePath - .resolve(context.containerName().asString() + "_" + DATE_TIME_FORMATTER.format(timer.currentTime()) + logsDirInContainer.pathInContainer()); - - // Files.move() does not support moving non-empty directories across providers, move using host paths - UnixPath containerLogsOnHost = new UnixPath(logsDirInContainer.pathOnHost()); - if (containerLogsOnHost.exists()) { - new UnixPath(containerLogsInArchiveDir).createParents(); - containerLogsOnHost.moveIfExists(containerLogsInArchiveDir); - } - new UnixPath(context.paths().of("/")).deleteRecursively(); - - // Operations on ContainerPath will fail if Container FS root doesn't exist, it is therefore important that - // it exists as long as NodeAgent is running. Normally the root is only created when NodeAgent is first - // started. Because non-tenant nodes are never removed from node-repo, we immediately re-create the new root - // after archiving the previous - if (context.nodeType() != NodeType.tenant) - context.paths().of("/").getFileSystem().createRoot(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java deleted file mode 100644 index 99715e6cad9..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.google.common.net.InetAddresses; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask; -import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; -import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; - -import java.io.IOException; -import java.net.InetAddress; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * This class maintains the iptables (ipv4 and ipv6) for all running containers. - * The filter table is synced with ACLs fetched from the Node repository while the nat table - * is synched with the proper redirect rule. - * <p> - * If an ACL cannot be configured (e.g. iptables process execution fails) we attempted to flush the rules - * rendering the firewall open. - * <p> - * This class currently assumes control over the filter and nat table. - * <p> - * The configuration will be retried the next time the maintainer runs. - * - * @author mpolden - * @author smorgrav - */ -public class AclMaintainer { - private static final Logger logger = Logger.getLogger(AclMaintainer.class.getName()); - - private final ContainerOperations containerOperations; - private final IPAddresses ipAddresses; - - public AclMaintainer(ContainerOperations containerOperations, IPAddresses ipAddresses) { - this.containerOperations = containerOperations; - this.ipAddresses = ipAddresses; - } - - // ip(6)tables operate while having the xtables lock, run with synchronized to prevent multiple NodeAgents - // invoking ip(6)tables concurrently. - public synchronized void converge(NodeAgentContext context) { - if (context.isDisabled(NodeAgentTask.AclMaintainer)) return; - - // Apply acl to the filter table - editFlushOnError(context, IPVersion.IPv4, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv4)); - editFlushOnError(context, IPVersion.IPv6, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv6)); - - ipAddresses.getAddress(context.hostname().value(), IPVersion.IPv4).ifPresent(addr -> applyRedirect(context, addr)); - ipAddresses.getAddress(context.hostname().value(), IPVersion.IPv6).ifPresent(addr -> applyRedirect(context, addr)); - } - - private void applyRedirect(NodeAgentContext context, InetAddress address) { - IPVersion ipVersion = IPVersion.get(address); - // Necessary to avoid the routing packets destined for the node's own public IP address - // via the bridge, which is illegal. - String redirectRule = "-A OUTPUT -d " + InetAddresses.toAddrString(address) + ipVersion.singleHostCidr() + " -j REDIRECT"; - editLogOnError(context, ipVersion, "nat", NatTableLineEditor.from(redirectRule)); - } - - private boolean editFlushOnError(NodeAgentContext context, IPVersion ipVersion, String table, LineEditor lineEditor) { - return edit(context, table, ipVersion, lineEditor, true); - } - - private boolean editLogOnError(NodeAgentContext context, IPVersion ipVersion, String table, LineEditor lineEditor) { - return edit(context, table, ipVersion, lineEditor, false); - } - - private boolean edit(NodeAgentContext context, String table, IPVersion ipVersion, LineEditor lineEditor, boolean flush) { - Editor editor = new Editor( - ipVersion.iptablesCmd() + "-" + table, - listTable(context, table, ipVersion), - restoreTable(context, table, ipVersion, flush), - lineEditor); - return editor.edit(message -> context.log(logger, message)); - } - - private Supplier<List<String>> listTable(NodeAgentContext context, String table, IPVersion ipVersion) { - return () -> containerOperations - .executeCommandInNetworkNamespace(context, new CommandLine.Options().setSilent(true), ipVersion.iptablesCmd(), "-S", "-t", table) - .mapEachLine(String::trim); - } - - private Consumer<List<String>> restoreTable(NodeAgentContext context, String table, IPVersion ipVersion, boolean flush) { - return list -> { - try (TemporaryIpTablesFileHandler fileHandler = new TemporaryIpTablesFileHandler(table)) { - String rules = String.join("\n", list); - String fileContent = "*" + table + "\n" + rules + "\nCOMMIT\n"; - fileHandler.writeUtf8Content(fileContent); - containerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesRestore(), fileHandler.absolutePath()); - } catch (Exception e) { - if (flush) { - context.log(logger, Level.SEVERE, "Exception occurred while syncing iptable " + table + ", attempting rollback", e); - try { - containerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesCmd(), "-F", "-t", table); - } catch (Exception ne) { - context.log(logger, Level.SEVERE, "Rollback of table " + table + " failed, giving up", ne); - } - } else { - context.log(logger, Level.WARNING, "Unable to sync iptables for " + table, e); - } - } - }; - } - - private static class TemporaryIpTablesFileHandler implements AutoCloseable { - private final Path path; - - private TemporaryIpTablesFileHandler(String table) { - this.path = uncheck(() -> Files.createTempFile("iptables-restore", "." + table)); - } - - private void writeUtf8Content(String content) throws IOException { - Files.writeString(path, content); - } - - private String absolutePath() { - return path.toAbsolutePath().toString(); - } - - @Override - public void close() throws IOException { - Files.deleteIfExists(path); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java deleted file mode 100644 index 4b831745f27..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit; -import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; - -import java.util.List; - -/** - * An editor that assumes all rules in the filter table are exactly as the wanted rules - * - * @author smorgrav - */ -class FilterTableLineEditor implements LineEditor { - - private final List<String> wantedRules; - private int position = 0; - - private FilterTableLineEditor(List<String> wantedRules) { - this.wantedRules = List.copyOf(wantedRules); - } - - static FilterTableLineEditor from(Acl acl, IPVersion ipVersion) { - List<String> rules = acl.toRules(ipVersion); - return new FilterTableLineEditor(rules); - } - - @Override - public LineEdit edit(String line) { - int index = indexOf(wantedRules, line, position); - // Unwanted rule, remove - if (index < 0) return LineEdit.remove(); - - // Wanted rule at the expected position, no diff - if (index == position) { - position++; - return LineEdit.none(); - } - - // Insert the rules between position and index before index - List<String> toInsert = wantedRules.subList(position, index); - position = ++index; - return LineEdit.insertBefore(toInsert); - } - - @Override - public List<String> onComplete() { - return wantedRules.subList(position, wantedRules.size()); - } - - private static <T> int indexOf(List<T> list, T value, int startPos) { - for (int i = startPos; i < list.size(); i++) { - if (value.equals(list.get(i))) - return i; - } - - return -1; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java deleted file mode 100644 index 9eff816d467..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit; -import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; - -import java.util.List; - -/** - * An editor that only cares about the REDIRECT statement - * - * @author smorgrav - */ -class NatTableLineEditor implements LineEditor { - - private final String redirectRule; - private boolean redirectExists; - - private NatTableLineEditor(String redirectRule) { - this.redirectRule = redirectRule; - } - - static NatTableLineEditor from(String redirectRule) { - return new NatTableLineEditor(redirectRule); - } - - @Override - public LineEdit edit(String line) { - if (line.endsWith("REDIRECT")) { - if (redirectExists) { - // Only allow one redirect rule - return LineEdit.remove(); - } else { - redirectExists = true; - if (line.equals(redirectRule)) { - return LineEdit.none(); - } else { - return LineEdit.replaceWith(redirectRule); - } - } - } else return LineEdit.none(); - } - - @Override - public List<String> onComplete() { - if (redirectExists) return List.of(); - return List.of(redirectRule); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java deleted file mode 100644 index f98a32ba488..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java deleted file mode 100644 index 0028784eec8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; - -import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; - -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Takes in an uncompressed core dump and collects relevant metadata. - * - * @author freva - */ -public class CoreCollector { - private static final Logger logger = Logger.getLogger(CoreCollector.class.getName()); - - private static final Pattern JAVA_HEAP_DUMP_PATTERN = Pattern.compile("java_pid.*\\.hprof$"); - private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("(?m)^Core was generated by `(?<path>.*?)'\\."); - private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'"); - private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'"); - static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-12/root/bin/gdb"; - - private final ContainerOperations container; - - public CoreCollector(ContainerOperations container) { - this.container = container; - } - - String readBinPathFallback(NodeAgentContext context, ContainerPath coredumpPath) { - String[] command = {GDB_PATH_RHEL8, "-n", "-batch", "-core", coredumpPath.pathInContainer()}; - CommandResult result = container.executeCommandInContainer(context, context.users().root(), command); - - Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getOutput()); - if (! matcher.find()) { - throw ConvergenceException.ofError(String.format("Failed to extract binary path from GDB, result: %s, command: %s", - asString(result), Arrays.toString(command))); - } - return matcher.group("path").split(" ")[0]; - } - - String readBinPath(NodeAgentContext context, ContainerPath coredumpPath) { - String[] command = {"file", coredumpPath.pathInContainer()}; - try { - CommandResult result = container.executeCommandInContainer(context, context.users().root(), command); - if (result.getExitCode() != 0) { - throw ConvergenceException.ofError("file command failed with " + asString(result)); - } - - Matcher execfnMatcher = EXECFN_PATH_PATTERN.matcher(result.getOutput()); - if (execfnMatcher.find()) { - return execfnMatcher.group("path").split(" ")[0]; - } - - Matcher fromMatcher = FROM_PATH_PATTERN.matcher(result.getOutput()); - if (fromMatcher.find()) { - return fromMatcher.group("path").split(" ")[0]; - } - } catch (RuntimeException e) { - context.log(logger, Level.WARNING, String.format("Failed getting bin path, command: %s. " + - "Trying fallback instead", Arrays.toString(command)), e); - } - - return readBinPathFallback(context, coredumpPath); - } - - List<String> readBacktrace(NodeAgentContext context, ContainerPath coredumpPath, String binPath, boolean allThreads) { - String threads = allThreads ? "thread apply all bt" : "bt"; - String[] command = {GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", threads, "-batch", binPath, coredumpPath.pathInContainer()}; - - CommandResult result = container.executeCommandInContainer(context, context.users().root(), command); - if (result.getExitCode() != 0) - throw ConvergenceException.ofError("Failed to read backtrace " + asString(result) + ", Command: " + Arrays.toString(command)); - - return List.of(result.getOutput().split("\n")); - } - - List<String> readJstack(NodeAgentContext context, ContainerPath coredumpPath, String binPath) { - String[] command = {"jhsdb", "jstack", "--exe", binPath, "--core", coredumpPath.pathInContainer()}; - - CommandResult result = container.executeCommandInContainer(context, context.users().root(), command); - if (result.getExitCode() != 0) - throw ConvergenceException.ofError("Failed to read jstack " + asString(result) + ", Command: " + Arrays.toString(command)); - - return List.of(result.getOutput().split("\n")); - } - - CoreDumpMetadata collect(NodeAgentContext context, ContainerPath coredumpPath) { - var metadata = new CoreDumpMetadata() - .setCreated(new UnixPath(coredumpPath).getLastModifiedTime()); - - if (JAVA_HEAP_DUMP_PATTERN.matcher(coredumpPath.getFileName().toString()).find()) { - metadata.setType(CoreDumpMetadata.Type.JVM_HEAP) - .setBinPath("java") - .setBacktrace(List.of("Heap dump, no backtrace available")); - return metadata; - } - - try { - String binPath = context.paths().underVespaHome("").resolve(readBinPath(context, coredumpPath)).pathInContainer(); - metadata.setType(CoreDumpMetadata.Type.CORE_DUMP).setBinPath(binPath); - - if (Path.of(binPath).getFileName().toString().equals("java")) { - metadata.setBacktraceAllThreads(readJstack(context, coredumpPath, binPath)); - } else { - metadata.setBacktrace(readBacktrace(context, coredumpPath, binPath, false)); - metadata.setBacktraceAllThreads(readBacktrace(context, coredumpPath, binPath, true)); - } - } catch (ConvergenceException e) { - context.log(logger, Level.WARNING, "Failed to extract backtrace: " + e.getMessage()); - } catch (RuntimeException e) { - context.log(logger, Level.WARNING, "Failed to extract backtrace", e); - } - return metadata; - } - - private String asString(CommandResult result) { - return "exit status " + result.getExitCode() + ", output '" + result.getOutput() + "'"; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java deleted file mode 100644 index a3386a3032f..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.Timer; -import com.yahoo.security.KeyId; -import com.yahoo.security.SecretSharedKey; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileDeleter; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileMover; -import com.yahoo.vespa.hosted.node.admin.task.util.file.MakeDirectory; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.IntStream; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameEndsWith; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameStartsWith; -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Finds coredumps, collects metadata and reports them - * - * @author freva - */ -public class CoredumpHandler { - - public static final String COREDUMP_FILENAME_PREFIX = "dump_"; - - private static final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); - private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log"); - private static final String PROCESSING_DIRECTORY_NAME = "processing"; - private static final String METADATA2_FILE_NAME = "metadata2.json"; - private static final String COMPRESSED_EXTENSION = ".zst"; - private static final String ENCRYPTED_EXTENSION = ".enc"; - - private final CoreCollector coreCollector; - private final Cores cores; - private final String crashPatchInContainer; - private final Path doneCoredumpsPath; - private final Metrics metrics; - private final Timer timer; - private final Supplier<String> coredumpIdSupplier; - private final SecretSharedKeySupplier secretSharedKeySupplier; - private final StringFlag coreEncryptionPublicKeyIdFlag; - - /** - * @param crashPathInContainer path inside the container where core dump are dumped - * @param doneCoredumpsPath path on host where processed core dumps are stored - */ - public CoredumpHandler(CoreCollector coreCollector, Cores cores, - String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, Timer timer, - SecretSharedKeySupplier secretSharedKeySupplier, FlagSource flagSource) { - this(coreCollector, cores, crashPathInContainer, doneCoredumpsPath, - metrics, timer, () -> UUID.randomUUID().toString(), secretSharedKeySupplier, - flagSource); - } - - CoredumpHandler(CoreCollector coreCollector, Cores cores, - String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, - Timer timer, Supplier<String> coredumpIdSupplier, - SecretSharedKeySupplier secretSharedKeySupplier, FlagSource flagSource) { - this.coreCollector = coreCollector; - this.cores = cores; - this.crashPatchInContainer = crashPathInContainer; - this.doneCoredumpsPath = doneCoredumpsPath; - this.metrics = metrics; - this.timer = timer; - this.coredumpIdSupplier = coredumpIdSupplier; - this.secretSharedKeySupplier = secretSharedKeySupplier; - this.coreEncryptionPublicKeyIdFlag = Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.bindTo(flagSource); - } - - - public void converge(NodeAgentContext context, Optional<DockerImage> dockerImage, boolean throwIfCoreBeingWritten) { - ContainerPath containerCrashPath = context.paths().of(crashPatchInContainer, context.users().vespa()); - ContainerPath containerProcessingPath = containerCrashPath.resolve(PROCESSING_DIRECTORY_NAME); - - updateMetrics(context, containerCrashPath); - - if (throwIfCoreBeingWritten) { - List<String> pendingCores = FileFinder.files(containerCrashPath) - .match(fileAttributes -> !isReadyForProcessing(fileAttributes)) - .maxDepth(1).stream() - .map(FileFinder.FileAttributes::filename) - .toList(); - if (!pendingCores.isEmpty()) - throw ConvergenceException.ofError(String.format("Cannot process %s coredumps: Still being written", - pendingCores.size() < 5 ? pendingCores : pendingCores.size())); - } - - // Check if we have already started to process a core dump or we can enqueue a new core one - getCoredumpToProcess(context, containerCrashPath, containerProcessingPath) - .ifPresent(path -> processAndReportSingleCoreDump(context, path, dockerImage)); - } - - /** @return path to directory inside processing directory that contains a core dump file to process */ - Optional<ContainerPath> getCoredumpToProcess(NodeAgentContext context, ContainerPath containerCrashPath, ContainerPath containerProcessingPath) { - return FileFinder.directories(containerProcessingPath).stream() - .map(FileFinder.FileAttributes::path) - .findAny() - .map(ContainerPath.class::cast) - .or(() -> enqueueCoredump(context, containerCrashPath, containerProcessingPath)); - } - - /** - * Moves a coredump and related hs_err file(s) to a new directory under the processing/ directory. - * Limit to only processing one coredump at the time, starting with the oldest. - * - * Assumption: hs_err files are much smaller than core files and are written (last modified time) - * before the core file. - * - * @return path to directory inside processing directory which contains the enqueued core dump file - */ - Optional<ContainerPath> enqueueCoredump(NodeAgentContext context, ContainerPath containerCrashPath, ContainerPath containerProcessingPath) { - Predicate<String> isCoreDump = filename -> !HS_ERR_PATTERN.matcher(filename).matches(); - - List<Path> toProcess = FileFinder.files(containerCrashPath) - .match(attributes -> { - if (isReadyForProcessing(attributes)) { - return true; - } else { - if (isCoreDump.test(attributes.filename())) - context.log(logger, attributes.path() + " is still being written"); - return false; - } - }) - .maxDepth(1) - .stream() - .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime)) - .map(FileFinder.FileAttributes::path) - .toList(); - - int coredumpIndex = IntStream.range(0, toProcess.size()) - .filter(i -> isCoreDump.test(toProcess.get(i).getFileName().toString())) - .findFirst() - .orElse(-1); - - // Either there are no files in crash directory, or all the files are hs_err files. - if (coredumpIndex == -1) return Optional.empty(); - - ContainerPath enqueuedDir = containerProcessingPath.resolve(coredumpIdSupplier.get()); - new MakeDirectory(enqueuedDir).createParents().converge(context); - IntStream.range(0, coredumpIndex + 1) - .forEach(i -> { - Path path = toProcess.get(i); - String prefix = i == coredumpIndex ? COREDUMP_FILENAME_PREFIX : ""; - new FileMover(path, enqueuedDir.resolve(prefix + path.getFileName())).converge(context); - }); - return Optional.of(enqueuedDir); - } - - private String corePublicKeyFlagValue(NodeAgentContext context) { - return coreEncryptionPublicKeyIdFlag.with(FetchVector.Dimension.NODE_TYPE, context.nodeType().name()).value(); - } - - static OutputStream wrapWithEncryption(OutputStream wrappedStream, SecretSharedKey sharedCoreKey) { - return sharedCoreKey.makeEncryptionCipher().wrapOutputStream(wrappedStream); - } - - /** - * Compresses and, if a key is provided, encrypts core file (and deletes the uncompressed core), then moves - * the entire core dump processing directory to {@link #doneCoredumpsPath} for archive - */ - private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory, SecretSharedKey sharedCoreKey) { - ContainerPath coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory); - String extension = COMPRESSED_EXTENSION + ENCRYPTED_EXTENSION; - ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + extension); - - try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(Files.newInputStream(coreFile)); - OutputStream fos = wrapWithEncryption(Files.newOutputStream(compressedCoreFile), sharedCoreKey)) { - zcis.transferTo(fos); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - new FileDeleter(coreFile).converge(context); - - Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString()); - new MakeDirectory(newCoredumpDirectory).createParents().converge(context); - // Files.move() does not support moving non-empty directories across providers, move using host paths - new FileMover(coredumpDirectory.pathOnHost(), newCoredumpDirectory.resolve(coredumpDirectory.getFileName().toString())) - .converge(context); - } - - ContainerPath findCoredumpFileInProcessingDirectory(ContainerPath coredumpProccessingDirectory) { - return (ContainerPath) FileFinder.files(coredumpProccessingDirectory) - .match(nameStartsWith(COREDUMP_FILENAME_PREFIX).and(nameEndsWith(COMPRESSED_EXTENSION).negate()) - .and(nameEndsWith(ENCRYPTED_EXTENSION).negate())) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::path) - .findFirst() - .orElseThrow(() -> new IllegalStateException( - "No coredump file found in processing directory " + coredumpProccessingDirectory)); - } - - void updateMetrics(NodeAgentContext context, ContainerPath containerCrashPath) { - Dimensions dimensions = generateDimensions(context); - - // Unprocessed coredumps - int numberOfUnprocessedCoredumps = FileFinder.files(containerCrashPath) - .match(nameStartsWith(".").negate()) - .match(nameMatches(HS_ERR_PATTERN).negate()) - .match(nameEndsWith(COMPRESSED_EXTENSION).negate()) - .match(nameEndsWith(ENCRYPTED_EXTENSION).negate()) - .match(nameStartsWith("metadata").negate()) - .list().size(); - - metrics.declareGauge(Metrics.APPLICATION_NODE, "coredumps.enqueued", dimensions, Metrics.DimensionType.PRETAGGED).sample(numberOfUnprocessedCoredumps); - - // Processed coredumps - Path processedCoredumpsPath = doneCoredumpsPath.resolve(context.containerName().asString()); - int numberOfProcessedCoredumps = FileFinder.directories(processedCoredumpsPath) - .maxDepth(1) - .list().size(); - - metrics.declareGauge(Metrics.APPLICATION_NODE, "coredumps.processed", dimensions, Metrics.DimensionType.PRETAGGED).sample(numberOfProcessedCoredumps); - } - - private Dimensions generateDimensions(NodeAgentContext context) { - NodeSpec node = context.node(); - Dimensions.Builder dimensionsBuilder = new Dimensions.Builder() - .add("host", node.hostname()) - .add("flavor", node.flavor()) - .add("state", node.state().toString()) - .add("zone", context.zone().getId().value()); - - node.owner().ifPresent(owner -> - dimensionsBuilder - .add("tenantName", owner.tenant().value()) - .add("applicationName", owner.application().value()) - .add("instanceName", owner.instance().value()) - .add("app", String.join(".", owner.application().value(), owner.instance().value())) - .add("applicationId", owner.toFullString()) - ); - - node.membership().ifPresent(membership -> - dimensionsBuilder - .add("clustertype", membership.type().value()) - .add("clusterid", membership.clusterId()) - ); - - node.parentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent)); - dimensionsBuilder.add("system", context.zone().getSystemName().value()); - - return dimensionsBuilder.build(); - } - - private boolean isReadyForProcessing(FileFinder.FileAttributes fileAttributes) { - // Wait at least a minute until we start processing a core/heap dump to ensure that - // kernel/JVM has finished writing it - return timer.currentTime().minusSeconds(60).isAfter(fileAttributes.lastModifiedTime()); - } - - void processAndReportSingleCoreDump(NodeAgentContext context, ContainerPath coreDumpDirectory, - Optional<DockerImage> dockerImage) { - CoreDumpMetadata metadata = gatherMetadata(context, coreDumpDirectory); - dockerImage.ifPresent(metadata::setDockerImage); - dockerImage.flatMap(DockerImage::tag).ifPresent(metadata::setVespaVersion); - dockerImage.ifPresent(metadata::setDockerImage); - SecretSharedKey sharedCoreKey = Optional.of(corePublicKeyFlagValue(context)) - .filter(k -> !k.isEmpty()) - .map(KeyId::ofString) - .flatMap(secretSharedKeySupplier::create) - .orElseThrow(() -> ConvergenceException.ofError("No core dump encryption key provided")); - metadata.setDecryptionToken(sharedCoreKey.sealedSharedKey().toTokenString()); - - String coreDumpId = coreDumpDirectory.getFileName().toString(); - cores.report(context.hostname(), coreDumpId, metadata); - context.log(logger, "Core dump reported: " + coreDumpId); - finishProcessing(context, coreDumpDirectory, sharedCoreKey); - } - - CoreDumpMetadata gatherMetadata(NodeAgentContext context, ContainerPath coreDumpDirectory) { - ContainerPath metadataPath = coreDumpDirectory.resolve(METADATA2_FILE_NAME); - Optional<ReportCoreDumpRequest> request = ReportCoreDumpRequest.load(metadataPath); - if (request.isPresent()) { - var metadata = new CoreDumpMetadata(); - request.get().populateMetadata(metadata, doneCoredumpsPath.getFileSystem()); - return metadata; - } - - ContainerPath coreDumpFile = findCoredumpFileInProcessingDirectory(coreDumpDirectory); - CoreDumpMetadata metadata = coreCollector.collect(context, coreDumpFile); - metadata.setCpuMicrocodeVersion(getMicrocodeVersion()) - .setKernelVersion(System.getProperty("os.version")) - .setCoreDumpPath(doneCoredumpsPath.resolve(context.containerName().asString()) - .resolve(coreDumpDirectory.getFileName().toString()) - .resolve(coreDumpFile.getFileName().toString())); - - ReportCoreDumpRequest requestInstance = new ReportCoreDumpRequest(); - requestInstance.fillFrom(metadata); - requestInstance.save(metadataPath); - context.log(logger, "Wrote " + metadataPath.pathOnHost()); - return metadata; - } - - private String getMicrocodeVersion() { - String output = uncheck(() -> Files.readAllLines(doneCoredumpsPath.getFileSystem().getPath("/proc/cpuinfo")).stream() - .filter(line -> line.startsWith("microcode")) - .findFirst() - .orElse("microcode : UNKNOWN")); - - String[] results = output.split(":"); - if (results.length != 2) { - throw ConvergenceException.ofError("Result from detect microcode command not as expected: " + output); - } - - return results[1].trim(); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/SecretSharedKeySupplier.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/SecretSharedKeySupplier.java deleted file mode 100644 index e5291c837a2..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/SecretSharedKeySupplier.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; - -import com.yahoo.security.KeyId; -import com.yahoo.security.SecretSharedKey; - -import java.util.Optional; - -/** - * @author vekterli - */ -@FunctionalInterface -public interface SecretSharedKeySupplier { - - Optional<SecretSharedKey> create(KeyId publicKeyId); - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/package-info.java deleted file mode 100644 index 0b6b3d18b01..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java deleted file mode 100644 index 50cd16f5617..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; - -import java.nio.file.Path; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.temporal.ChronoField; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameStartsWith; - -/** - * @author freva - */ -public class CoredumpCleanupRule { - - private static final Comparator<FileAttributes> CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR = Comparator - .comparing((FileAttributes fa) -> !fa.filename().contains("vespa-")) - .thenComparing(FileAttributes::lastModifiedTime); - - public static DiskCleanupRule forContainer(Path containerCrashPath) { - return new ContainerCoredumpCleanupRule(containerCrashPath); - } - - public static DiskCleanupRule forHost(Path processedCoredumpsPath) { - return new HostCoredumpCleanupRule(processedCoredumpsPath); - } - - /** Assigns MEDIUM priority to the oldest, unprocessed coredump and HIGHEST for the remaining */ - private static class ContainerCoredumpCleanupRule implements DiskCleanupRule { - private final Path containerCrashPath; - - private ContainerCoredumpCleanupRule(Path containerCrashPath) { - this.containerCrashPath = containerCrashPath; - } - - @Override - public Collection<PrioritizedFileAttributes> prioritize() { - List<FileAttributes> fileAttributes = FileFinder.files(containerCrashPath) - .maxDepth(1).stream() - .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR) - .toList(); - - return mapFirstAndRemaining(fileAttributes, Priority.MEDIUM, Priority.HIGHEST).toList(); - } - } - - /** Assigns MEDIUM priority to the first coredump of the day for each container, HIGH for the remaining */ - private static class HostCoredumpCleanupRule implements DiskCleanupRule { - private final Path processedCoredumpsPath; - - private HostCoredumpCleanupRule(Path processedCoredumpsPath) { - this.processedCoredumpsPath = processedCoredumpsPath; - } - - @Override - public Collection<PrioritizedFileAttributes> prioritize() { - Map<String, List<FileAttributes>> fileAttributesByContainerDay = FileFinder.files(processedCoredumpsPath) - .match(nameStartsWith(CoredumpHandler.COREDUMP_FILENAME_PREFIX)) - .stream() - .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR) - .collect(Collectors.groupingBy( - // Group FileAttributes by string [container-name]_[day of year], e.g. zt00534-v6-2_234 - fa -> containerNameFromProcessedCoredumpPath(fa.path()) + "_" + dayOfYear(fa.lastModifiedTime()), - Collectors.collectingAndThen( - Collectors.toCollection(ArrayList::new), - l -> { l.sort(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR); return l; } ))); - - return fileAttributesByContainerDay.values().stream() - .flatMap(fa -> mapFirstAndRemaining(fa, Priority.MEDIUM, Priority.HIGH)) - .toList(); - } - } - - /** - * Maps list of FileAttributes into list of PrioritizedFileAttributes where the first FileAttribute is given - * {@code first} priority, while the remaining FileAttributes are given {@code remaining} priority */ - private static Stream<PrioritizedFileAttributes> mapFirstAndRemaining(List<FileAttributes> fileAttributes, Priority first, Priority remaining) { - return IntStream.range(0, fileAttributes.size()) - .mapToObj(i -> new PrioritizedFileAttributes(fileAttributes.get(i), i == 0 ? first : remaining)); - } - - /** Extracts container-name from path under processed-coredumps or empty string */ - private static String containerNameFromProcessedCoredumpPath(Path path) { - if (path.getNameCount() < 3) return ""; // Path is too short - return path.getName(path.getNameCount() - 3).toString(); - } - - /** Returns day number of the year (1-365 (or 366 for leap years)) */ - private static int dayOfYear(Instant instant) { - return instant.atOffset(ZoneOffset.UTC).get(ChronoField.DAY_OF_YEAR); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java deleted file mode 100644 index 54cf9324909..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.logging.Logger; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author freva - */ -public class DiskCleanup { - - private static final Logger logger = Logger.getLogger(DiskCleanup.class.getName()); - private static final Comparator<PrioritizedFileAttributes> PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR = Comparator - .comparing(PrioritizedFileAttributes::priority) - .thenComparingLong(f -> f.fileAttributes().size()) - .reversed(); - - public boolean cleanup(TaskContext context, List<DiskCleanupRule> rules, long bytesToRemove) { - if (bytesToRemove <= 0) return false; - - long[] btr = new long[] { bytesToRemove }; - List<Path> deletedPaths = new ArrayList<>(); - try { - rules.stream() - .flatMap(rule -> rule.prioritize().stream()) - .sorted(PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR) - .takeWhile(fa -> btr[0] > 0) - .forEach(pfa -> { - if (uncheck(() -> Files.deleteIfExists(pfa.fileAttributes().path()))) { - btr[0] -= pfa.fileAttributes().size(); - deletedPaths.add(pfa.fileAttributes().path()); - } - }); - - } finally { - String wantedDeleteSize = DiskSize.of(bytesToRemove).asString(); - String deletedSize = DiskSize.of(bytesToRemove - btr[0]).asString(); - if (deletedPaths.size() > 20) { - context.log(logger, "Deleted %d files (%s) because disk was getting full", deletedPaths.size(), deletedSize); - } else if (deletedPaths.size() > 0) { - context.log(logger, "Deleted %s because disk was getting full from: %s", deletedSize, deletedPaths); - } else { - context.log(logger, "Wanted to delete %s, but failed to find any files to delete", wantedDeleteSize); - } - } - - return !deletedPaths.isEmpty(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java deleted file mode 100644 index 88b89f1f201..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; - -import java.util.Collection; - -/** - * @author freva - */ -public interface DiskCleanupRule { - - Collection<PrioritizedFileAttributes> prioritize(); - - enum Priority { - LOWEST, LOW, MEDIUM, HIGH, HIGHEST - } - - record PrioritizedFileAttributes(FileFinder.FileAttributes fileAttributes, Priority priority) { } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java deleted file mode 100644 index 961d978bfcf..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; - -/** - * Prioritizes files by first scoring them with the given scoring function and then mapping the scores to a - * priority within the given range. - * The priority room is evenly split between given lowest and highest priority for range [0, 1.0). Scores below 0 - * are assigned lowest, while scores at or higher than 1 are assigned highest priority. - * - * Typical use-case is for log files. The scoring function calculates the file age and normalizes it by dividing it - * by expected max age of log files. The oldest logs will then by prioritized by highest given priority. - * - * @author freva - */ -public class LinearCleanupRule implements DiskCleanupRule { - private final Supplier<List<FileAttributes>> lister; - private final Function<FileAttributes, Priority> prioritizer; - - public LinearCleanupRule(Supplier<List<FileAttributes>> lister, - Function<FileAttributes, Double> scorer, Priority lowest, Priority highest) { - if (lowest.ordinal() > highest.ordinal()) - throw new IllegalArgumentException("Lowest priority: " + lowest + " is higher than highest priority: " + highest); - - this.lister = lister; - - Priority[] values = Priority.values(); - int range = highest.ordinal() - lowest.ordinal() + 1; - this.prioritizer = fa -> { - int ordinal = (int) (lowest.ordinal() + scorer.apply(fa) * range); - return values[Math.max(lowest.ordinal(), Math.min(highest.ordinal(), ordinal))]; - }; - } - - @Override - public Collection<PrioritizedFileAttributes> prioritize() { - return lister.get().stream() - .map(fa -> new PrioritizedFileAttributes(fa, prioritizer.apply(fa))) - .toList(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java deleted file mode 100644 index 6b5f60a66c7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java deleted file mode 100644 index f46950aa448..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.identity; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.jdisc.Timer; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; -import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.athenz.client.zts.ZtsClientException; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.client.CsrGenerator; -import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient; -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.vespa.athenz.utils.SiaUtils; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.NODE; -import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.TENANT; - -/** - * A maintainer that is responsible for providing and refreshing Athenz credentials for a container. - * - * @author bjorncs - */ -public class AthenzCredentialsMaintainer implements CredentialsMaintainer { - - private static final Logger logger = Logger.getLogger(AthenzCredentialsMaintainer.class.getName()); - - private static final Duration EXPIRY_MARGIN = Duration.ofDays(1); - private static final Duration REFRESH_PERIOD = Duration.ofDays(1); - private static final Duration REFRESH_BACKOFF = Duration.ofHours(1); // Backoff when refresh fails to ensure ZTS is not DDoS'ed. - - private static final String CONTAINER_SIA_DIRECTORY = "/var/lib/sia"; - private static final String LEGACY_SIA_DIRECTORY = "/opt/vespa/var/vespa/sia"; - - private final Path ztsTrustStorePath; - private final Timer timer; - private final String certificateDnsSuffix; - private final ServiceIdentityProvider hostIdentityProvider; - private final IdentityDocumentClient identityDocumentClient; - - // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts - private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); - - public AthenzCredentialsMaintainer(Path ztsTrustStorePath, - ConfigServerInfo configServerInfo, - String certificateDnsSuffix, - ServiceIdentityProvider hostIdentityProvider, - Timer timer) { - this.ztsTrustStorePath = ztsTrustStorePath; - this.certificateDnsSuffix = certificateDnsSuffix; - this.hostIdentityProvider = hostIdentityProvider; - this.identityDocumentClient = new DefaultIdentityDocumentClient( - configServerInfo.getLoadBalancerEndpoint(), - hostIdentityProvider, - new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity()))); - this.timer = timer; - } - - public boolean converge(NodeAgentContext context) { - var modified = false; - modified |= maintain(context, NODE); - - if (context.zone().getSystemName().isPublic()) - return modified; - - modified |= maintain(context, TENANT); - return modified; - } - - private boolean maintain(NodeAgentContext context, IdentityType identityType) { - if (context.isDisabled(NodeAgentTask.CredentialsMaintainer)) return false; - - try { - var modified = false; - context.log(logger, Level.FINE, "Checking certificate"); - ContainerPath siaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa()); - ContainerPath identityDocumentFile = siaDirectory.resolve(identityType.getIdentityDocument()); - Optional<AthenzIdentity> optionalAthenzIdentity = getAthenzIdentity(context, identityType, identityDocumentFile); - if (optionalAthenzIdentity.isEmpty()) - return false; - AthenzIdentity athenzIdentity = optionalAthenzIdentity.get(); - ContainerPath privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(siaDirectory, athenzIdentity); - ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(siaDirectory, athenzIdentity); - if (!Files.exists(privateKeyFile) || !Files.exists(certificateFile) || !Files.exists(identityDocumentFile)) { - context.log(logger, "Certificate/private key/identity document file does not exist"); - Files.createDirectories(privateKeyFile.getParent()); - Files.createDirectories(certificateFile.getParent()); - Files.createDirectories(identityDocumentFile.getParent()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity); - modified = true; - } - - X509Certificate certificate = readCertificateFromFile(certificateFile); - Instant now = timer.currentTime(); - Instant expiry = certificate.getNotAfter().toInstant(); - var doc = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile); - if (refreshIdentityDocument(doc, context)) { - context.log(logger, "Identity document is outdated (version=%d)", doc.documentVersion()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity); - modified = true; - } else if (isCertificateExpired(expiry, now)) { - context.log(logger, "Certificate has expired (expiry=%s)", expiry.toString()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity); - modified = true; - } - - Duration age = Duration.between(certificate.getNotBefore().toInstant(), now); - if (shouldRefreshCredentials(age)) { - context.log(logger, "Certificate is ready to be refreshed (age=%s)", age.toString()); - if (shouldThrottleRefreshAttempts(context.containerName(), now)) { - context.log(logger, Level.WARNING, String.format( - "Skipping refresh attempt as last refresh was on %s (less than %s ago)", - lastRefreshAttempt.get(context.containerName()).toString(), REFRESH_BACKOFF.toString())); - } else { - lastRefreshAttempt.put(context.containerName(), now); - refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc.identityDocument(), identityType, athenzIdentity); - modified = true; - } - } - - if (identityType == TENANT) { - modified |= maintainRoleCertificates(context, siaDirectory, privateKeyFile, certificateFile, athenzIdentity, doc.identityDocument()); - copyCredsToLegacyPath(context, privateKeyFile, certificateFile); - } - return modified; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private boolean maintainRoleCertificates(NodeAgentContext context, - ContainerPath siaDirectory, - ContainerPath privateKeyFile, - ContainerPath certificateFile, - AthenzIdentity identity, - IdentityDocument identityDocument) { - var modified = false; - - for (var role : getRoleList(context)) { - try { - var roleCertificatePath = siaDirectory.resolve("certs") - .resolve(String.format("%s.cert.pem", role)); - var roleKeyPath = siaDirectory.resolve("keys") - .resolve(String.format("%s.key.pem", role)); - if (Files.notExists(roleCertificatePath)) { - writeRoleCredentials(context, privateKeyFile, certificateFile, roleCertificatePath, roleKeyPath, identity, identityDocument, role); - modified = true; - } else if (shouldRefreshCertificate(context, roleCertificatePath)) { - writeRoleCredentials(context, privateKeyFile, certificateFile, roleCertificatePath, roleKeyPath, identity, identityDocument, role); - modified = true; - } - } catch (IOException e) { - context.log(logger, Level.WARNING, "Failed to maintain role certificate " + role, e); - } - } - return modified; - } - - private boolean shouldRefreshCertificate(NodeAgentContext context, ContainerPath certificatePath) throws IOException { - var certificate = readCertificateFromFile(certificatePath); - var now = timer.currentTime(); - var shouldRefresh = now.isAfter(certificate.getNotAfter().toInstant()) || - now.isAfter(certificate.getNotBefore().toInstant().plus(REFRESH_PERIOD)); - return !shouldThrottleRefreshAttempts(context.containerName(), now) && - shouldRefresh; - } - - private void writeRoleCredentials(NodeAgentContext context, - ContainerPath privateKeyFile, - ContainerPath certificateFile, - ContainerPath roleCertificatePath, - ContainerPath roleKeyPath, - AthenzIdentity identity, - IdentityDocument identityDocument, - String role) throws IOException { - HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; - var keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - var athenzRole = AthenzRole.fromResourceNameString(role); - - try (ZtsClient ztsClient = ztsClient(identityDocument.ztsUrl(), privateKeyFile, certificateFile, ztsHostNameVerifier)) { - var csrGenerator = new CsrGenerator(certificateDnsSuffix, identityDocument.providerService().getFullName()); - var csr = csrGenerator.generateRoleCsr( - identity, athenzRole, identityDocument.providerUniqueId(), identityDocument.clusterType(), keyPair); - var roleCertificate = ztsClient.getRoleCertificate(athenzRole, csr); - writePrivateKeyAndCertificate(roleKeyPath, keyPair.getPrivate(), roleCertificatePath, roleCertificate); - context.log(logger, "Role certificate successfully retrieved written to file " + roleCertificatePath.pathInContainer()); - } - } - - private boolean refreshIdentityDocument(SignedIdentityDocument signedIdentityDocument, NodeAgentContext context) { - int expectedVersion = documentVersion(context); - return signedIdentityDocument.outdated() || signedIdentityDocument.documentVersion() != expectedVersion; - } - - public void clearCredentials(NodeAgentContext context) { - FileFinder.files(context.paths().of(CONTAINER_SIA_DIRECTORY)) - .deleteRecursively(context); - lastRefreshAttempt.remove(context.containerName()); - } - - @Override - public Duration certificateLifetime(NodeAgentContext context) { - ContainerPath containerSiaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY); - ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(containerSiaDirectory, context.identity()); - try { - X509Certificate certificate = readCertificateFromFile(certificateFile); - Instant now = timer.currentTime(); - Instant expiry = certificate.getNotAfter().toInstant(); - return Duration.between(now, expiry); - } catch (IOException e) { - context.log(logger, Level.SEVERE, "Unable to read certificate at " + certificateFile, e); - return Duration.ZERO; - } - } - - @Override - public String name() { - return "node-certificate"; - } - - private boolean shouldRefreshCredentials(Duration age) { - return age.compareTo(REFRESH_PERIOD) >= 0; - } - - private boolean shouldThrottleRefreshAttempts(ContainerName containerName, Instant now) { - return REFRESH_BACKOFF.compareTo( - Duration.between( - lastRefreshAttempt.getOrDefault(containerName, Instant.EPOCH), - now)) > 0; - } - - private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, IdentityType identityType, AthenzIdentity identity) { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - SignedIdentityDocument signedDoc = signedIdentityDocument(context, identityType); - IdentityDocument doc = signedDoc.identityDocument(); - CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); - Pkcs10Csr csr = csrGenerator.generateInstanceCsr( - identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); - - // Allow all zts hosts while removing SIS - HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; - try (ZtsClient ztsClient = ztsClient(doc.ztsUrl(), hostIdentityProvider.privateKeyPath(), hostIdentityProvider.certificatePath(), ztsHostNameVerifier)) { - InstanceIdentity instanceIdentity = - ztsClient.registerInstance( - doc.providerService(), - identity, - EntityBindingsMapper.toAttestationData(signedDoc), - csr); - EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, signedDoc); - writePrivateKeyAndCertificate(privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate()); - context.log(logger, "Instance successfully registered and credentials written to file"); - } - } - - private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, - ContainerPath identityDocumentFile, IdentityDocument doc, IdentityType identityType, AthenzIdentity identity) { - try { - // Do not rotate private key on every refresh. - // TODO: rotate key pair only on Vespa upgrade or similar - PrivateKey privateKey = readPrivateKeyFromFile(privateKeyFile); - KeyPair keyPair = KeyUtils.toKeyPair(privateKey); - CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); - Pkcs10Csr csr = csrGenerator.generateInstanceCsr( - identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); - - // Allow all zts hosts while removing SIS - HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; - try (ZtsClient ztsClient = ztsClient(doc.ztsUrl(), privateKeyFile, certificateFile, ztsHostNameVerifier)) { - InstanceIdentity instanceIdentity = - ztsClient.refreshInstance( - doc.providerService(), - identity, - doc.providerUniqueId().asDottedString(), - csr); - writePrivateKeyAndCertificate(privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate()); - context.log(logger, "Instance successfully refreshed and credentials written to file"); - } catch (ZtsClientException e) { - if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) { - context.log(logger, Level.SEVERE, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, identity); - } else { - throw e; - } - } - } catch (Exception e) { - context.log(logger, Level.SEVERE, "Certificate refresh failed: " + e.getMessage(), e); - } - } - - - private static void writePrivateKeyAndCertificate(ContainerPath privateKeyFile, - PrivateKey privateKey, - ContainerPath certificateFile, - X509Certificate certificate) { - writeFile(privateKeyFile, KeyUtils.toPem(privateKey)); - writeFile(certificateFile, X509CertificateUtils.toPem(certificate)); - } - - private static void writeFile(ContainerPath path, String utf8Content) { - new UnixPath(path.resolveSibling(path.getFileName() + ".tmp")) - .writeUtf8File(utf8Content, "r--------") - .atomicMove(path); - } - - private static X509Certificate readCertificateFromFile(ContainerPath certificateFile) throws IOException { - String pemEncodedCertificate = new String(Files.readAllBytes(certificateFile)); - return X509CertificateUtils.fromPem(pemEncodedCertificate); - } - - private static PrivateKey readPrivateKeyFromFile(ContainerPath privateKeyFile) throws IOException { - String pemEncodedKey = new String(Files.readAllBytes(privateKeyFile)); - return KeyUtils.fromPemEncodedPrivateKey(pemEncodedKey); - } - - private static boolean isCertificateExpired(Instant expiry, Instant now) { - return now.isAfter(expiry.minus(EXPIRY_MARGIN)); - } - - private SignedIdentityDocument signedIdentityDocument(NodeAgentContext context, IdentityType identityType) { - return switch (identityType) { - case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value(), documentVersion(context)); - case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value(), documentVersion(context)).get(); - }; - } - - private Optional<AthenzIdentity> getAthenzIdentity(NodeAgentContext context, IdentityType identityType, ContainerPath identityDocumentFile) { - return switch (identityType) { - case NODE -> Optional.of(context.identity()); - case TENANT -> getTenantIdentity(context, identityDocumentFile); - }; - } - - private Optional<AthenzIdentity> getTenantIdentity(NodeAgentContext context, ContainerPath identityDocumentFile) { - if (Files.exists(identityDocumentFile)) { - return Optional.of(EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).identityDocument().serviceIdentity()); - } else { - return identityDocumentClient.getTenantIdentityDocument(context.hostname().value(), documentVersion(context)) - .map(doc -> doc.identityDocument().serviceIdentity()); - } - } - - private void copyCredsToLegacyPath(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile) throws IOException { - var legacySiaDirectory = context.paths().of(LEGACY_SIA_DIRECTORY, context.users().vespa()); - var keysDirectory = legacySiaDirectory.resolve("keys"); - var certsDirectory = legacySiaDirectory.resolve("certs"); - Files.createDirectories(keysDirectory); - Files.createDirectories(certsDirectory); - writeFile(certsDirectory.resolve(certificateFile.getFileName()), Files.readString(certificateFile)); - writeFile(keysDirectory.resolve(privateKeyFile.getFileName()), Files.readString(privateKeyFile)); - } - - /** Get the document version to ask for */ - private int documentVersion(NodeAgentContext context) { - return SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; - } - - private ZtsClient ztsClient(URI ztsEndpoint, Path privateKeyFile, Path certificateFile, HostnameVerifier hostnameVerifier) { - SSLContext sslContext = new SslContextBuilder() - .withKeyStore(privateKeyFile, certificateFile) - .withTrustStore(ztsTrustStorePath) - .build(); - return new DefaultZtsClient.Builder(ztsEndpoint) - .withSslContext(sslContext) - .withHostnameVerifier(hostnameVerifier) - .build(); - } - - private List<String> getRoleList(NodeAgentContext context) { - try { - return identityDocumentClient.getNodeRoles(context.hostname().value()); - } catch (Exception e) { - context.log(logger, Level.WARNING, "Failed to retrieve role list", e); - return List.of(); - } - } - - enum IdentityType { - NODE("vespa-node-identity-document.json"), - TENANT("vespa-tenant-identity-document.json"); - - private final String identityDocument; - IdentityType(String identityDocument) { - this.identityDocument = identityDocument; - } - - public String getIdentityDocument() { - return identityDocument; - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/CredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/CredentialsMaintainer.java deleted file mode 100644 index 0e387ac2731..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/CredentialsMaintainer.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.identity; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; - -import java.time.Duration; - -/** - * A maintainer that is responsible for providing and refreshing credentials for a container. - * - * @author freva - */ -public interface CredentialsMaintainer { - - /** - * Creates/refreshes credentials for the given NodeAgentContext. Called for every NodeAgent tick. - * @return false if already converged, i.e. was a no-op. - */ - boolean converge(NodeAgentContext context); - - /** Remove any existing credentials. This method is called just before container data is archived. */ - void clearCredentials(NodeAgentContext context); - - /** Get time until the certificate expires. Invoked each time metrics are collected. */ - Duration certificateLifetime(NodeAgentContext context); - - /** Name used when reporting metrics */ - String name(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java deleted file mode 100644 index a48f4f45aa8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author bjorncs - */ -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.maintenance.identity; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java deleted file mode 100644 index 2ef78aa9c54..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/Artifact.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/Artifact.java deleted file mode 100644 index ee8da84e9cb..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/Artifact.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import java.util.Optional; - -/** - * An artifact file produced by a {@link ArtifactProducer}. - * - * @author bjorncs - */ -class Artifact { - - enum Classification { - CONFIDENTIAL("confidential"), - INTERNAL("internal"); - - private final String value; - Classification(String value) { this.value = value; } - public String value() { return value; } - } - - private final Classification classification; - private final ContainerPath file; - private final boolean compressOnUpload; - - private Artifact(Builder builder) { - if (builder.file == null) { - throw new IllegalArgumentException("No file specified"); - } - this.file = builder.file; - this.classification = builder.classification; - this.compressOnUpload = Boolean.TRUE.equals(builder.compressOnUpload); - } - - static Builder newBuilder() { return new Builder(); } - - Optional<Classification> classification() { return Optional.ofNullable(classification); } - ContainerPath file() { return file; } - boolean compressOnUpload() { return compressOnUpload; } - - static class Builder { - private Classification classification; - private ContainerPath file; - private Boolean compressOnUpload; - - private Builder() {} - - Builder classification(Classification c) { this.classification = c; return this; } - Builder file(ContainerPath f) { this.file = f; return this; } - Builder compressOnUpload() { this.compressOnUpload = true; return this; } - Artifact build() { return new Artifact(this); } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducer.java deleted file mode 100644 index 87ab1ef8bf5..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducer.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; - -import java.util.List; -import java.util.OptionalDouble; - -/** - * Produces service dump artifacts. - * - * @author bjorncs - */ -interface ArtifactProducer { - - String artifactName(); - String description(); - List<Artifact> produceArtifacts(Context ctx); - - interface Context { - String serviceId(); - int servicePid(); - CommandResult executeCommandInNode(List<String> command, boolean logOutput); - ContainerPath outputContainerPath(); - ContainerPath containerPathUnderVespaHome(String relativePath); - Options options(); - - interface Options { - OptionalDouble duration(); - boolean callGraphRecording(); - boolean sendProfilingSignal(); - } - } - - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java deleted file mode 100644 index 939bebc5fac..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducers.java +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.yolean.concurrent.Sleeper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * @author bjorncs - */ -class ArtifactProducers { - - private final Map<String, ArtifactProducer> producers; - private final Map<String, List<ArtifactProducer>> aliases; - - private ArtifactProducers(Set<ArtifactProducer> producers, - Map<String, List<Class<? extends ArtifactProducer>>> aliases) { - var producerMap = producers.stream() - .collect(Collectors.toMap(ArtifactProducer::artifactName, Function.identity())); - Map<String, List<ArtifactProducer>> aliasMap = new HashMap<>(); - aliases.forEach((alias, mapping) -> { - List<ArtifactProducer> concreteMapping = mapping.stream() - .map(type -> producers.stream() - .filter(p -> p.getClass().equals(type)) - .findAny() - .orElseThrow(() -> new IllegalArgumentException("No producer of type " + type))) - .toList(); - if (producerMap.containsKey(alias)) { - throw new IllegalStateException("Alias name '" + alias + "' conflicts with producer"); - } - aliasMap.put(alias, concreteMapping); - }); - this.producers = producerMap; - this.aliases = aliasMap; - } - - static ArtifactProducers createDefault(Sleeper sleeper) { - var producers = Set.of( - new PerfReporter(), - new JvmDumper.JavaFlightRecorder(sleeper), - new JvmDumper.HeapDump(), - new JvmDumper.Jmap(), - new JvmDumper.Jstat(), - new JvmDumper.Jstack(), - new PmapReporter(), - new VespaLogDumper(sleeper), - new ZooKeeperSnapshotDumper(), - new ConfigDumper()); - var aliases = - Map.of( - "jvm-dump", - List.of( - JvmDumper.HeapDump.class, JvmDumper.Jmap.class, JvmDumper.Jstat.class, - JvmDumper.Jstack.class, VespaLogDumper.class) - ); - return new ArtifactProducers(producers, aliases); - } - - static ArtifactProducers createCustom(Set<ArtifactProducer> producers, - Map<String, List<Class<? extends ArtifactProducer>>> aliases) { - return new ArtifactProducers(producers, aliases); - } - - List<ArtifactProducer> resolve(List<String> requestedArtifacts) { - List<ArtifactProducer> resolved = new ArrayList<>(); - for (String artifact : requestedArtifacts) { - if (aliases.containsKey(artifact)) { - aliases.get(artifact).stream() - .filter(p -> !resolved.contains(p)) - .forEach(resolved::add); - } else if (producers.containsKey(artifact)) { - ArtifactProducer producer = producers.get(artifact); - if (!resolved.contains(producer)) { - resolved.add(producer); - } - } else { - throw createInvalidArtifactException(artifact); - } - } - return resolved; - } - - private IllegalArgumentException createInvalidArtifactException(String artifact) { - String producersString = producers.keySet().stream() - .map(a -> "'" + a + "'") - .sorted() - .collect(Collectors.joining(", ", "[", "]")); - String aliasesString = aliases.entrySet().stream() - .map(e -> String.format( - "'%s': %s", - e.getKey(), - e.getValue().stream() - .map(p -> "'" + p.artifactName() + "'") - .sorted() - .collect(Collectors.joining(", ", "[", "]"))) - ) - .collect(Collectors.joining(", ", "[", "]")); - String msg = String.format( - "Invalid artifact type '%s'. Valid types are %s and valid aliases are %s", - artifact, producersString, aliasesString); - return new IllegalArgumentException(msg); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ConfigDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ConfigDumper.java deleted file mode 100644 index 8eadabf07cf..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ConfigDumper.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL; -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.INTERNAL; - -/** - * Performs dump of config on a node. - * - * @author hmusum - */ -class ConfigDumper implements ArtifactProducer { - @Override public String artifactName() { return "config-dump"; } - @Override public String description() { return "Dumps config"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - ContainerPath dir = ctx.outputContainerPath().resolve("config"); - ContainerPath configDump = ctx.outputContainerPath().resolve("config-dump.tar.zst"); - List<String> cmd = List.of("bash", "-c", - String.format("mkdir -p %s; /opt/vespa/bin/vespa-configproxy-cmd -m dumpcache %s; tar cvf %s.tar %s; zstd %s.tar -o %s", - dir.pathInContainer(), - dir.pathInContainer(), - dir.pathInContainer(), - dir.pathInContainer(), - dir.pathInContainer(), - configDump.pathInContainer())); - ctx.executeCommandInNode(cmd, true); - return List.of(Artifact.newBuilder().classification(INTERNAL).file(configDump).build()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java deleted file mode 100644 index 360a212646f..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.yolean.concurrent.Sleeper; - -import java.time.Duration; -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL; -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.INTERNAL; - -/** - * @author bjorncs - */ -class JvmDumper { - private JvmDumper() {} - - static class HeapDump implements ArtifactProducer { - @Override public String artifactName() { return "jvm-heap-dump"; } - @Override public String description() { return "JVM heap dump"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - ContainerPath heapDumpFile = ctx.outputContainerPath().resolve("jvm-heap-dump.hprof"); - List<String> cmd = List.of( - "jmap", "-dump:live,format=b,file=" + heapDumpFile.pathInContainer(), Integer.toString(ctx.servicePid())); - ctx.executeCommandInNode(cmd, true); - return List.of( - Artifact.newBuilder().classification(CONFIDENTIAL).file(heapDumpFile).compressOnUpload().build()); - } - } - - static class Jmap implements ArtifactProducer { - @Override public String artifactName() { return "jvm-jmap"; } - @Override public String description() { return "JVM jmap output"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - ContainerPath jmapReport = ctx.outputContainerPath().resolve("jvm-jmap.txt"); - List<String> cmd = List.of("bash", "-c", "jhsdb jmap --heap --pid " + ctx.servicePid() + " > " + jmapReport.pathInContainer()); - ctx.executeCommandInNode(cmd, true); - return List.of(Artifact.newBuilder().classification(INTERNAL).file(jmapReport).build()); - } - } - - static class Jstat implements ArtifactProducer { - @Override public String artifactName() { return "jvm-jstat"; } - @Override public String description() { return "JVM jstat output"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - ContainerPath jstatReport = ctx.outputContainerPath().resolve("jvm-jstat.txt"); - List<String> cmd = List.of("bash", "-c", "jstat -gcutil " + ctx.servicePid() + " > " + jstatReport.pathInContainer()); - ctx.executeCommandInNode(cmd, true); - return List.of(Artifact.newBuilder().classification(INTERNAL).file(jstatReport).build()); - } - } - - static class Jstack implements ArtifactProducer { - @Override public String artifactName() { return "jvm-jstack"; } - @Override public String description() { return "JVM jstack output"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - ContainerPath jstackReport = ctx.outputContainerPath().resolve("jvm-jstack.txt"); - ctx.executeCommandInNode(List.of("bash", "-c", "jstack " + ctx.servicePid() + " > " + jstackReport.pathInContainer()), true); - return List.of(Artifact.newBuilder().classification(INTERNAL).file(jstackReport).build()); - } - } - - static class JavaFlightRecorder implements ArtifactProducer { - private final Sleeper sleeper; - - JavaFlightRecorder(Sleeper sleeper) { this.sleeper = sleeper; } - - @Override public String artifactName() { return "jvm-jfr"; } - @Override public String description() { return "Java Flight Recorder recording"; } - - @Override - public List<Artifact> produceArtifacts(ArtifactProducer.Context ctx) { - int seconds = (int) (ctx.options().duration().orElse(30.0)); - ContainerPath outputFile = ctx.outputContainerPath().resolve("recording.jfr"); - List<String> startCommand = List.of("jcmd", Integer.toString(ctx.servicePid()), "JFR.start", "name=host-admin", - "path-to-gc-roots=true", "settings=profile", "filename=" + outputFile.pathInContainer(), "duration=" + seconds + "s"); - ctx.executeCommandInNode(startCommand, true); - sleeper.sleep(Duration.ofSeconds(seconds).plusSeconds(1)); - int maxRetries = 10; - List<String> checkCommand = List.of("jcmd", Integer.toString(ctx.servicePid()), "JFR.check", "name=host-admin"); - for (int i = 0; i < maxRetries; i++) { - boolean stillRunning = ctx.executeCommandInNode(checkCommand, true).getOutputLines().stream() - .anyMatch(l -> l.contains("name=host-admin") && l.contains("running")); - if (!stillRunning) { - Artifact a = Artifact.newBuilder() - .classification(CONFIDENTIAL).file(outputFile).compressOnUpload().build(); - return List.of(a); - } - sleeper.sleep(Duration.ofSeconds(1)); - } - throw new RuntimeException("Failed to wait for JFR dump to complete after " + maxRetries + " retries"); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReporter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReporter.java deleted file mode 100644 index f4b4307b0d7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PerfReporter.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import java.util.ArrayList; -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL; -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.INTERNAL; - -/** - * @author bjorncs - */ -class PerfReporter implements ArtifactProducer { - - PerfReporter() {} - - @Override public String artifactName() { return "perf-report"; } - @Override public String description() { return "Perf recording and report"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - int duration = (int)ctx.options().duration().orElse(30.0); - List<String> perfRecordCommand = new ArrayList<>(List.of("perf", "record")); - if (ctx.options().callGraphRecording()) { - perfRecordCommand.add("-g"); - } - ContainerPath recordFile = ctx.outputContainerPath().resolve("perf-record.bin"); - perfRecordCommand.addAll( - List.of("--output=" + recordFile.pathInContainer(), - "--pid=" + ctx.servicePid(), "sleep", Integer.toString(duration))); - ctx.executeCommandInNode(perfRecordCommand, true); - ContainerPath reportFile = ctx.outputContainerPath().resolve("perf-report.txt"); - ctx.executeCommandInNode(List.of("bash", "-c", "perf report --input=" + recordFile.pathInContainer() + " > " + reportFile.pathInContainer()), true); - return List.of( - Artifact.newBuilder().classification(CONFIDENTIAL).file(recordFile).compressOnUpload().build(), - Artifact.newBuilder().classification(INTERNAL).file(reportFile).build()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PmapReporter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PmapReporter.java deleted file mode 100644 index 8f8feb57c27..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/PmapReporter.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.INTERNAL; - -/** - * @author bjorncs - */ -class PmapReporter implements ArtifactProducer { - @Override public String artifactName() { return "pmap"; } - @Override public String description() { return "Pmap report"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - ContainerPath pmapReport = ctx.outputContainerPath().resolve("pmap.txt"); - List<String> cmd = List.of("bash", "-c", "pmap -x " + ctx.servicePid() + " > " + pmapReport.pathInContainer()); - ctx.executeCommandInNode(cmd, true); - return List.of(Artifact.newBuilder().classification(INTERNAL).file(pmapReport).build()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java deleted file mode 100644 index 744eeefca07..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport; - -import java.net.URI; -import java.time.Instant; -import java.util.List; - -/** - * JSON representation of Vespa service dump report. - * - * @author bjorncs - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -class ServiceDumpReport extends BaseReport { - - public static final String REPORT_ID = "serviceDump"; - - private static final String STARTED_AT_FIELD = "startedAt"; - private static final String COMPLETED_AT_FIELD = "completedAt"; - private static final String FAILED_AT_FIELD = "failedAt"; - private static final String LOCATION_FIELD = "location"; - private static final String CONFIG_ID_FIELD = "configId"; - private static final String EXPIRE_AT_FIELD = "expireAt"; - private static final String ERROR_FIELD = "error"; - private static final String ARTIFACTS_FIELD = "artifacts"; - private static final String DUMP_OPTIONS_FIELD = "dumpOptions"; - - private final Long startedAt; - private final Long completedAt; - private final Long failedAt; - private final String location; - private final String configId; - private final Long expireAt; - private final String error; - private final List<String> artifacts; - private final DumpOptions dumpOptions; - - @JsonCreator - public ServiceDumpReport(@JsonProperty(CREATED_FIELD) Long createdAt, - @JsonProperty(STARTED_AT_FIELD) Long startedAt, - @JsonProperty(COMPLETED_AT_FIELD) Long completedAt, - @JsonProperty(FAILED_AT_FIELD) Long failedAt, - @JsonProperty(LOCATION_FIELD) String location, - @JsonProperty(CONFIG_ID_FIELD) String configId, - @JsonProperty(EXPIRE_AT_FIELD) Long expireAt, - @JsonProperty(ERROR_FIELD) String error, - @JsonProperty(ARTIFACTS_FIELD) List<String> artifacts, - @JsonProperty(DUMP_OPTIONS_FIELD) DumpOptions dumpOptions) { - super(createdAt, null); - this.startedAt = startedAt; - this.completedAt = completedAt; - this.failedAt = failedAt; - this.location = location; - this.configId = configId; - this.expireAt = expireAt; - this.error = error; - this.artifacts = artifacts; - this.dumpOptions = dumpOptions; - } - - public static ServiceDumpReport createRequestReport(Instant createdAt, Instant expireAt, String configId, - List<String> artifacts, DumpOptions options) { - return new ServiceDumpReport( - createdAt.toEpochMilli(), null, null, null, null, configId, - expireAt != null ? expireAt.toEpochMilli() : null, null, artifacts, options); - } - - public static ServiceDumpReport createStartedReport(ServiceDumpReport request, Instant startedAt) { - return new ServiceDumpReport( - request.getCreatedMillisOrNull(), startedAt.toEpochMilli(), null, null, null, request.configId(), - request.expireAt(), null, request.artifacts(), request.dumpOptions()); - } - - public static ServiceDumpReport createSuccessReport( - ServiceDumpReport request, Instant startedAt, Instant completedAt, URI location) { - return new ServiceDumpReport( - request.getCreatedMillisOrNull(), startedAt.toEpochMilli(), completedAt.toEpochMilli(), null, - location.toString(), request.configId(), request.expireAt(), null, request.artifacts(), - request.dumpOptions()); - } - - public static ServiceDumpReport createErrorReport( - ServiceDumpReport reqOrNull, Instant startedAt, Instant failedAt, String message) { - Long createdAt = reqOrNull != null ? reqOrNull.getCreatedMillisOrNull() : Long.valueOf(startedAt.toEpochMilli()); - String configId = reqOrNull != null ? reqOrNull.configId() : "unknown"; - Long expireAt = reqOrNull != null ? reqOrNull.expireAt() : null; - List<String> artifacts = reqOrNull != null ? reqOrNull.artifacts() : List.of(); - DumpOptions dumpOptions = reqOrNull != null ? reqOrNull.dumpOptions() : null; - return new ServiceDumpReport( - createdAt, startedAt.toEpochMilli(), null, failedAt.toEpochMilli(), null, - configId, expireAt, message, artifacts, dumpOptions); - } - - @JsonGetter(STARTED_AT_FIELD) public Long startedAt() { return startedAt; } - @JsonGetter(COMPLETED_AT_FIELD) public Long completedAt() { return completedAt; } - @JsonGetter(FAILED_AT_FIELD) public Long failedAt() { return failedAt; } - @JsonGetter(LOCATION_FIELD) public String location() { return location; } - @JsonGetter(CONFIG_ID_FIELD) public String configId() { return configId; } - @JsonGetter(EXPIRE_AT_FIELD) public Long expireAt() { return expireAt; } - @JsonGetter(ERROR_FIELD) public String error() { return error; } - @JsonGetter(ARTIFACTS_FIELD) public List<String> artifacts() { return artifacts; } - @JsonGetter(DUMP_OPTIONS_FIELD) public DumpOptions dumpOptions() { return dumpOptions; } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonIgnoreProperties(ignoreUnknown = true) - public static class DumpOptions { - - private static final String CALL_GRAPH_RECORDING_FIELD = "callGraphRecording"; - private static final String DURATION_FIELD = "duration"; - private static final String SEND_PROFILING_SIGNAL_FIELD = "sendProfilingSignal"; - - private final Boolean callGraphRecording; - private final Double duration; - private final Boolean sendProfilingSignal; - - @JsonCreator - public DumpOptions(@JsonProperty(CALL_GRAPH_RECORDING_FIELD) Boolean callGraphRecording, - @JsonProperty(DURATION_FIELD) Double duration, - @JsonProperty(SEND_PROFILING_SIGNAL_FIELD) Boolean sendProfilingSignal) { - this.callGraphRecording = callGraphRecording; - this.duration = duration; - this.sendProfilingSignal = sendProfilingSignal; - } - - @JsonGetter(CALL_GRAPH_RECORDING_FIELD) public Boolean callGraphRecording() { return callGraphRecording; } - @JsonGetter(DURATION_FIELD) public Double duration() { return duration; } - @JsonGetter(SEND_PROFILING_SIGNAL_FIELD) public Boolean sendProfilingSignal() { return sendProfilingSignal; } - } - - @JsonIgnore public boolean isCompletedOrFailed() { return !isNullTimestamp(failedAt) || !isNullTimestamp(completedAt); } - - public static boolean isNullTimestamp(Long timestamp) { return timestamp == null || timestamp == 0; } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaLogDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaLogDumper.java deleted file mode 100644 index 32814e38d39..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaLogDumper.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.yolean.concurrent.Sleeper; - -import java.nio.file.Files; -import java.time.Duration; -import java.util.List; -import java.util.logging.Logger; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL; -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author bjorncs - */ -class VespaLogDumper implements ArtifactProducer { - - private static final Logger log = Logger.getLogger(VespaLogDumper.class.getName()); - - private final Sleeper sleeper; - - VespaLogDumper(Sleeper sleeper) { this.sleeper = sleeper; } - - @Override public String artifactName() { return "vespa-log"; } - @Override public String description() { return "Current Vespa logs"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - if (ctx.options().sendProfilingSignal()) { - log.info("Sending SIGPROF to process to include vespa-malloc dump in Vespa log"); - ctx.executeCommandInNode(List.of("kill", "-SIGPROF", Integer.toString(ctx.servicePid())), true); - sleeper.sleep(Duration.ofSeconds(3)); - } - ContainerPath vespaLogFile = ctx.containerPathUnderVespaHome("logs/vespa/vespa.log"); - ContainerPath destination = ctx.outputContainerPath().resolve("vespa.log"); - if (Files.exists(vespaLogFile)) { - uncheck(() -> Files.copy(vespaLogFile, destination)); - return List.of( - Artifact.newBuilder().classification(CONFIDENTIAL).file(destination).compressOnUpload().build()); - } else { - log.info("Log file '" + vespaLogFile + "' does not exist"); - return List.of(); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumper.java deleted file mode 100644 index 1f474295660..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumper.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; - -/** - * @author bjorncs - */ -public interface VespaServiceDumper { - void processServiceDumpRequest(NodeAgentContext context); - - VespaServiceDumper DUMMY_INSTANCE = context -> {}; -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java deleted file mode 100644 index 1279d9a4b28..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java +++ /dev/null @@ -1,269 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudName; -import com.yahoo.jdisc.Timer; -import com.yahoo.text.Lowercase; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.yolean.concurrent.Sleeper; - -import java.io.UncheckedIOException; -import java.net.URI; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.ServiceDumpReport.isNullTimestamp; - -/** - * Generates dumps for Vespa services and uploads resulting files to S3. - * - * @author bjorncs - */ -public class VespaServiceDumperImpl implements VespaServiceDumper { - - private static final Logger log = Logger.getLogger(VespaServiceDumperImpl.class.getName()); - - private final ContainerOperations container; - private final SyncClient syncClient; - private final NodeRepository nodeRepository; - private final Timer timer; - private final ArtifactProducers artifactProducers; - - public VespaServiceDumperImpl(ContainerOperations container, SyncClient syncClient, NodeRepository nodeRepository, Timer timer) { - this(ArtifactProducers.createDefault(Sleeper.DEFAULT), container, syncClient, nodeRepository, timer); - } - - // For unit testing - VespaServiceDumperImpl(ArtifactProducers producers, ContainerOperations container, SyncClient syncClient, - NodeRepository nodeRepository, Timer timer) { - this.container = container; - this.syncClient = syncClient; - this.nodeRepository = nodeRepository; - this.timer = timer; - this.artifactProducers = producers; - } - - @Override - public void processServiceDumpRequest(NodeAgentContext context) { - if (context.zone().getCloudName().equals(CloudName.GCP)) return; - - Instant startedAt = timer.currentTime(); - NodeSpec nodeSpec = context.node(); - ServiceDumpReport request; - try { - request = nodeSpec.reports().getReport(ServiceDumpReport.REPORT_ID, ServiceDumpReport.class) - .orElse(null); - } catch (IllegalArgumentException | UncheckedIOException e) { - handleFailure(context, null, startedAt, e, "Invalid JSON in service dump request"); - return; - } - if (request == null || request.isCompletedOrFailed()) { - context.log(log, Level.FINE, "No service dump requested or dump already completed/failed"); - return; - } - if (isNullTimestamp(request.getCreatedMillisOrNull())) { - handleFailure(context, request, startedAt, "'createdMillis' is missing or null"); - return; - } - String configId = request.configId(); - if (configId == null) { - handleFailure(context, request, startedAt, "Service config id is missing from request"); - return; - } - Instant expiry = expireAt(startedAt, request); - if (expiry.isBefore(startedAt)) { - handleFailure(context, request, startedAt, "Request already expired"); - return; - } - List<String> requestedArtifacts = request.artifacts(); - if (requestedArtifacts == null || requestedArtifacts.isEmpty()) { - handleFailure(context, request, startedAt, "No artifacts requested"); - return; - } - ContainerPath directory = context.paths().underVespaHome("var/tmp/vespa-service-dump-" + request.getCreatedMillisOrNull()); - UnixPath unixPathDirectory = new UnixPath(directory); - try { - context.log(log, Level.INFO, - "Creating service dump for " + configId + " requested at " - + Instant.ofEpochMilli(request.getCreatedMillisOrNull())); - storeReport(context, ServiceDumpReport.createStartedReport(request, startedAt)); - if (unixPathDirectory.exists()) { - context.log(log, Level.INFO, "Removing existing directory '" + unixPathDirectory +"'."); - unixPathDirectory.deleteRecursively(); - } - context.log(log, Level.INFO, "Creating '" + unixPathDirectory +"'."); - unixPathDirectory.createDirectory("rwxr-x---"); - URI destination = serviceDumpDestination(nodeSpec, createDumpId(request)); - ProducerContext producerCtx = new ProducerContext(context, directory, request); - List<Artifact> producedArtifacts = new ArrayList<>(); - for (ArtifactProducer producer : artifactProducers.resolve(requestedArtifacts)) { - context.log(log, "Producing artifact of type '" + producer.artifactName() + "'"); - producedArtifacts.addAll(producer.produceArtifacts(producerCtx)); - } - uploadArtifacts(context, destination, producedArtifacts); - storeReport(context, ServiceDumpReport.createSuccessReport(request, startedAt, timer.currentTime(), destination)); - } catch (Exception e) { - handleFailure(context, request, startedAt, e, e.getMessage()); - } finally { - if (unixPathDirectory.exists()) { - context.log(log, Level.INFO, "Deleting directory '" + unixPathDirectory +"'."); - unixPathDirectory.deleteRecursively(); - } - } - } - - private void uploadArtifacts(NodeAgentContext ctx, URI destination, - List<Artifact> producedArtifacts) { - ApplicationId owner = ctx.node().owner().orElseThrow(); - List<SyncFileInfo> filesToUpload = producedArtifacts.stream() - .map(a -> { - Compression compression = a.compressOnUpload() ? Compression.ZSTD : Compression.NONE; - String classification = a.classification().map(Artifact.Classification::value).orElse(null); - return SyncFileInfo.forServiceDump(destination, a.file(), compression, owner, classification); - }) - .toList(); - ctx.log(log, Level.INFO, - "Uploading " + filesToUpload.size() + " file(s) with destination " + destination); - if (!syncClient.sync(ctx, filesToUpload, Integer.MAX_VALUE)) { - throw new RuntimeException("Unable to upload all files"); - } - ctx.log(log, Level.INFO, "Upload complete"); - } - - private static Instant expireAt(Instant startedAt, ServiceDumpReport request) { - return isNullTimestamp(request.expireAt()) - ? startedAt.plus(7, ChronoUnit.DAYS) - : Instant.ofEpochMilli(request.expireAt()); - } - - private void handleFailure(NodeAgentContext context, ServiceDumpReport requestOrNull, Instant startedAt, - Exception failure, String message) { - context.log(log, Level.WARNING, failure.toString(), failure); - ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, timer.currentTime(), message); - storeReport(context, report); - } - - private void handleFailure(NodeAgentContext context, ServiceDumpReport requestOrNull, Instant startedAt, String message) { - context.log(log, Level.WARNING, message); - ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, timer.currentTime(), message); - storeReport(context, report); - } - - private void storeReport(NodeAgentContext context, ServiceDumpReport report) { - NodeAttributes nodeAttributes = new NodeAttributes(); - nodeAttributes.withReport(ServiceDumpReport.REPORT_ID, report.toJsonNode()); - nodeRepository.updateNodeAttributes(context.hostname().value(), nodeAttributes); - } - - static String createDumpId(ServiceDumpReport request) { - String sanitizedConfigId = Lowercase.toLowerCase(request.configId()).replaceAll("[^a-z_0-9]", "-"); - return sanitizedConfigId + "-" + request.getCreatedMillisOrNull().toString(); - } - - private static URI serviceDumpDestination(NodeSpec spec, String dumpId) { - URI archiveUri = spec.archiveUri() - .orElseThrow(() -> new IllegalStateException("Archive URI is missing for " + spec.hostname())); - String targetDirectory = "service-dump/" + dumpId + "/"; - return archiveUri.resolve(targetDirectory); - } - - private class ProducerContext implements ArtifactProducer.Context, ArtifactProducer.Context.Options { - - final NodeAgentContext nodeAgentCtx; - final ContainerPath path; - final ServiceDumpReport request; - volatile int pid = -1; - - ProducerContext(NodeAgentContext nodeAgentCtx, ContainerPath path, ServiceDumpReport request) { - this.nodeAgentCtx = nodeAgentCtx; - this.path = path; - this.request = request; - } - - @Override public String serviceId() { return request.configId(); } - - @Override - public int servicePid() { - if (pid == -1) { - pid = findServicePid(serviceId()); - } - return pid; - } - - private int findServicePid(String serviceId) { - ContainerPath findPidBinary = nodeAgentCtx.paths().underVespaHome("libexec/vespa/find-pid"); - CommandResult findPidResult = executeCommandInNode(List.of(findPidBinary.pathInContainer(), serviceId), true); - return Integer.parseInt(findPidResult.getOutput()); - } - - @Override - public CommandResult executeCommandInNode(List<String> command, boolean logOutput) { - CommandResult result = container.executeCommandInContainer(nodeAgentCtx, nodeAgentCtx.users().vespa(), command.toArray(new String[0])); - String cmdString = command.stream().map(s -> "'" + s + "'").collect(Collectors.joining(" ", "\"", "\"")); - int exitCode = result.getExitCode(); - String output = result.getOutput().trim(); - String prefixedOutput = output.contains("\n") - ? "\n" + output - : (output.isEmpty() ? "<no output>" : output); - if (exitCode > 0) { - String errorMsg = logOutput - ? String.format("Failed to execute %s (exited with code %d): %s", cmdString, exitCode, prefixedOutput) - : String.format("Failed to execute %s (exited with code %d)", cmdString, exitCode); - throw new RuntimeException(errorMsg); - } else { - String logMsg = logOutput - ? String.format("Executed command %s. Exited with code %d and output: %s", cmdString, exitCode, prefixedOutput) - : String.format("Executed command %s. Exited with code %d.", cmdString, exitCode); - nodeAgentCtx.log(log, logMsg); - } - return result; - } - - @Override public ContainerPath outputContainerPath() { return path; } - - @Override - public ContainerPath containerPathUnderVespaHome(String relativePath) { - return nodeAgentCtx.paths().underVespaHome(relativePath); - } - - @Override public Options options() { return this; } - - @Override - public OptionalDouble duration() { - Double duration = dumpOptions() - .map(ServiceDumpReport.DumpOptions::duration) - .orElse(null); - return duration != null ? OptionalDouble.of(duration) : OptionalDouble.empty(); - } - - @Override - public boolean callGraphRecording() { - return dumpOptions().map(ServiceDumpReport.DumpOptions::callGraphRecording).orElse(false); - } - - @Override - public boolean sendProfilingSignal() { - return dumpOptions().map(ServiceDumpReport.DumpOptions::sendProfilingSignal).orElse(false); - } - - Optional<ServiceDumpReport.DumpOptions> dumpOptions() { return Optional.ofNullable(request.dumpOptions()); } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java deleted file mode 100644 index c8f930464e0..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ZooKeeperSnapshotDumper.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Artifact.Classification.CONFIDENTIAL; - -/** - * Performs dump of ZooKeeper snapshots. Can be used for controllers, config servers, cluster controllers and tenant containers - * where zookeeper is configured. - * - * @author hmusum - */ -class ZooKeeperSnapshotDumper implements ArtifactProducer { - @Override public String artifactName() { return "zookeeper-snapshot"; } - @Override public String description() { return "Dumps ZooKeeper snapshots"; } - - @Override - public List<Artifact> produceArtifacts(Context ctx) { - ContainerPath zookeeperSnapshot = ctx.outputContainerPath().resolve("zookeeper-snapshot.tgz"); - List<String> cmd = List.of("bash", "-c", String.format("/opt/vespa/bin/vespa-backup-zk-data.sh -o %s -k -f", zookeeperSnapshot.pathInContainer())); - ctx.executeCommandInNode(cmd, true); - return List.of(Artifact.newBuilder().classification(CONFIDENTIAL).file(zookeeperSnapshot).build()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/package-info.java deleted file mode 100644 index 3ea43b6129a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author bjorncs - */ -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java deleted file mode 100644 index b1e467ad446..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.sync; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.util.List; - -/** - * @author freva - */ -public interface SyncClient { - - /** - * Syncs the given files, will only upload each file once. - * - * @param context context used to log which files were synced - * @param syncFileInfos list of files and their metadata to sync - * @param limit max number of files to upload for this invocation, to avoid blocking for too long - * @return true iff any files were uploaded - */ - boolean sync(TaskContext context, List<SyncFileInfo> syncFileInfos, int limit); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java deleted file mode 100644 index c65f2abb6fd..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.sync; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.net.URI; -import java.nio.file.Path; -import java.time.Duration; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; - -/** - * @author freva - */ -public class SyncFileInfo { - - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter - .ofPattern("yyyy-MM-dd.HH-mm-ss").withZone(ZoneOffset.UTC); - - private final Path source; - private final Function<String, URI> destination; - private final Compression uploadCompression; - private final Map<String, String> tags; - private final Optional<Duration> minDurationBetweenSync; - - private SyncFileInfo(Path source, Function<String, URI> destination, Compression uploadCompression, - Map<String, String> tags, Duration minDurationBetweenSyncOrNull) { - this.source = source; - this.destination = destination; - this.uploadCompression = uploadCompression; - this.tags = Map.copyOf(tags); - this.minDurationBetweenSync = Optional.ofNullable(minDurationBetweenSyncOrNull); - } - - /** Source path of the file to sync */ - public Path source() { - return source; - } - - /** Remote URI to store the file at */ - public URI destination() { - return destination.apply(""); - } - - /** Returns a destination URI after adding a suffix to the base name of the filename. */ - public URI destinationWithBasenameSuffix(String suffix) { - return destination.apply(suffix); - } - - /** Compression algorithm to use when uploading the file */ - public Compression uploadCompression() { - return uploadCompression; - } - - public Map<String, String> tags() { return tags; } - - public Optional<Duration> minDurationBetweenSync() { return minDurationBetweenSync; } - - public static Optional<SyncFileInfo> forLogFile(URI uri, Path logFile, boolean rotatedOnly, ApplicationId owner) { - String filename = logFile.getFileName().toString(); - Compression compression; - final String dir; - String remoteFilename = logFile.getFileName().toString(); - Duration minDurationBetweenSync = null; - - if (filename.startsWith("vespa.log")) { - dir = "logs/vespa/"; - compression = Compression.ZSTD; - if (filename.length() == 9) { - if (!rotatedOnly) remoteFilename = "vespa.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); - minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO; - } - } else if (filename.startsWith("zookeeper.") && filename.endsWith(".log")) { - compression = Compression.ZSTD; - dir = "logs/zookeeper/"; - remoteFilename = rotatedOnly && filename.endsWith(".0.log") ? "zookeeper.log" : - "zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); - minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; - } else if (filename.startsWith("start-services.out-")) { - compression = Compression.ZSTD; - dir = "logs/start-services/"; - } else if (filename.startsWith("nginx-error")) { - compression = Compression.ZSTD; - if ("nginx-error.log".equals(filename)) { - if (!rotatedOnly) remoteFilename = "nginx-error.log"; - minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO; - } - dir = "logs/nginx/"; - } else { - compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD; - if (rotatedOnly && compression != Compression.NONE) - dir = null; - else if (filename.contains(".metrics-proxy.")) // See AccessLogComponent.java for filename. - dir = null; - else if (filename.startsWith("JsonAccessLog.") || filename.startsWith("access")) - dir = "logs/access/"; - else if (filename.startsWith("ConnectionLog.")) - dir = "logs/connection/"; - else - dir = null; - } - - if (dir == null) return Optional.empty(); - String finalRemoteFilename = remoteFilename; - Function<String, URI> destination = suffix -> uri.resolve(dir + finalRemoteFilename + suffix + compression.extension); - return Optional.of(new SyncFileInfo(logFile, destination, compression, defaultTags(owner), minDurationBetweenSync)); - } - - public static SyncFileInfo forServiceDump(URI destinationDir, Path file, Compression compression, - ApplicationId owner, String assetClassification) { - String filename = file.getFileName().toString(); - Function<String, URI> location = suffix -> destinationDir.resolve(filename + suffix + compression.extension); - Map<String, String> tags = defaultTags(owner); - if (assetClassification != null) { - tags.put("vespa:AssetClassification", assetClassification); - } - return new SyncFileInfo(file, location, compression, tags, null); - } - - private static Map<String, String> defaultTags(ApplicationId owner) { - var tags = new HashMap<String, String>(); - tags.put("corp:Application", owner.toFullString()); - return tags; - } - - public boolean overwriteIfExists() { - return minDurationBetweenSync.isPresent(); - } - - public enum Compression { - NONE(""), ZSTD(".zst"); - - private final String extension; - Compression(String extension) { - this.extension = extension; - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java deleted file mode 100644 index eeec7e1b59c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.sync; - -import com.yahoo.compress.ZstdCompressor; - -import java.io.IOException; -import java.io.InputStream; - -/** - * InputStream that outputs given InputStream compressed with the ZStandard. - * - * @author freva - */ -public class ZstdCompressingInputStream extends InputStream { - - public static final int DEFAULT_INPUT_BUFFER_SIZE = 8 * 1024; - private final ZstdCompressor compressor = new ZstdCompressor(); - - private final InputStream is; - private final byte[] inputBuffer; - private final byte[] outputBuffer; - - private boolean firstRead = true; - private boolean eof = false; - private int outputPosition = 0; - private int outputLength = 0; - private boolean isClosed = false; - - public ZstdCompressingInputStream(InputStream is, int inputBufferSize) { - this.is = is; - this.inputBuffer = new byte[inputBufferSize]; - this.outputBuffer = new byte[ZstdCompressor.getMaxCompressedLength(inputBufferSize)]; - } - - public ZstdCompressingInputStream(InputStream is) { - this(is, DEFAULT_INPUT_BUFFER_SIZE); - } - - @Override - public int read() throws IOException { - throwIfClosed(); - - if (outputPosition >= outputLength) { - int readLength = eof ? -1 : is.read(inputBuffer); - if (readLength == -1) { - if (!firstRead) - return -1; - // zstd compressing an empty file results in a 13 bytes file. - eof = true; - readLength = 0; - } - firstRead = false; - - outputLength = compressor.compress(inputBuffer, 0, readLength, outputBuffer, 0, outputBuffer.length); - outputPosition = 0; - } - - return Byte.toUnsignedInt(outputBuffer[outputPosition++]); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int first = read(); - if (first == -1) return -1; - - b[off++] = (byte) first; - len = Math.min(Math.min(len, outputLength - outputPosition), b.length - off); - System.arraycopy(outputBuffer, outputPosition, b, off, len); - outputPosition += len; - return len + 1; - } - - @Override - public void close() throws IOException { - throwIfClosed(); - is.close(); - isClosed = true; - } - - private void throwIfClosed() { - if (isClosed) throw new IllegalArgumentException("Input stream is already closed"); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java deleted file mode 100644 index becf11945e3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author freva - */ -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.maintenance.sync; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ConvergenceException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ConvergenceException.java deleted file mode 100644 index 686c32fd5ee..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ConvergenceException.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -/** - * Exception specially handled to avoid dumping full stack trace on convergence failure. - * - * @author hakonhall - */ -@SuppressWarnings("serial") -public class ConvergenceException extends RuntimeException { - /** Create an exception that will NOT increment the monitored unhandled_exceptions metric. */ - public static ConvergenceException ofTransient(String message) { return ofTransient(message, null); } - - /** Create an exception that will NOT increment the monitored unhandled_exceptions metric. */ - public static ConvergenceException ofTransient(String message, Throwable t) { return new ConvergenceException(message, t, false); } - - /** Create an exception that increments the monitored unhandled_exceptions metric. */ - public static ConvergenceException ofError(String message) { return ofError(message, null); } - - /** Create an exception that increments the monitored unhandled_exceptions metric. */ - public static ConvergenceException ofError(String message, Throwable t) { return new ConvergenceException(message, t, true); } - - /** Create an exception with the same transient/error as the cause. */ - public static ConvergenceException ofNested(String message, ConvergenceException cause) { return new ConvergenceException(message, cause, cause.isError); } - - private final boolean isError; - - /** @param isError whether the exception should increment the monitored unhandled_exception metric. */ - protected ConvergenceException(String message, boolean isError) { - this(message, null, isError); - } - - /** @param isError whether the exception should increment the monitored unhandled_exception metric. */ - protected ConvergenceException(String message, Throwable t, boolean isError) { - super(message, t); - this.isError = isError; - } - - /** Whether the exception signals an error someone may want to look at, or whether it is expected to be transient (false). */ - public boolean isError() { return isError; } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java deleted file mode 100644 index 986f6b4eebc..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; - -import java.time.Duration; -import java.util.Set; - -/** - * NodeAdmin manages the life cycle of NodeAgents. - * @author Haakon Dybdahl - */ -public interface NodeAdmin { - - /** Start/stop NodeAgents and schedule next NodeAgent ticks with the given NodeAgentContexts */ - void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts); - - /** Update node admin metrics */ - void updateMetrics(boolean isSuspended); - - /** - * Attempts to freeze/unfreeze all NodeAgents and itself. To freeze a NodeAgent means that - * they will not pick up any changes from NodeRepository. - * - * @param frozen whether NodeAgents and NodeAdmin should be frozen - * @return True if all the NodeAgents and NodeAdmin has converged to the desired state - */ - boolean setFrozen(boolean frozen); - - /** - * Returns whether NodeAdmin itself is currently frozen, meaning it will not pick up any changes - * from NodeRepository. - */ - boolean isFrozen(); - - /** - * Returns an upper bound on the time some or all parts of the node admin (including agents) - * have been frozen. Returns 0 if not frozen, nor trying to freeze. - */ - Duration subsystemFreezeDuration(); - - /** - * Stop all services on these nodes - */ - void stopNodeAgentServices(); - - /** - * Start node-admin schedulers. - */ - void start(); - - /** - * Stop the NodeAgents. Will not delete the storage or stop the container. - */ - void stop(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java deleted file mode 100644 index 446f21d53e7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import ai.vespa.metrics.ContainerMetrics; -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.hosted.node.admin.container.ContainerStats; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Counter; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Gauge; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextManager; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentScheduler; - -import java.nio.file.FileSystem; -import java.time.Duration; -import java.time.Instant; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Administers a host (for now only docker hosts) and its nodes (docker containers nodes). - * - * @author stiankri - */ -public class NodeAdminImpl implements NodeAdmin { - private static final Duration NODE_AGENT_FREEZE_TIMEOUT = Duration.ofSeconds(5); - private static final Duration NODE_AGENT_SPREAD = Duration.ofSeconds(3); - - private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory; - - private final Timer timer; - private final Duration freezeTimeout; - private final Duration spread; - private boolean previousWantFrozen; - private boolean isFrozen; - private Instant startOfFreezeConvergence; - private final Map<String, NodeAgentWithScheduler> nodeAgentWithSchedulerByHostname = new ConcurrentHashMap<>(); - - private final ProcMeminfoReader procMeminfoReader; - private final Gauge jvmHeapUsed; - private final Gauge jvmHeapFree; - private final Gauge jvmHeapTotal; - private final Gauge containerCount; - private final Counter numberOfUnhandledExceptions; - private final Metrics metrics; - private Dimensions previousMemoryOverheadDimensions = null; - - public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Timer timer, FileSystem fileSystem) { - this(nodeAgentContext -> create(timer, nodeAgentFactory, nodeAgentContext), - metrics, timer, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD, new ProcMeminfoReader(fileSystem)); - } - - public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, - Timer timer, Duration freezeTimeout, Duration spread, ProcMeminfoReader procMeminfoReader) { - this(nodeAgentContext -> create(timer, nodeAgentFactory, nodeAgentContext), - metrics, timer, freezeTimeout, spread, procMeminfoReader); - } - - NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory, - Metrics metrics, Timer timer, Duration freezeTimeout, Duration spread, - ProcMeminfoReader procMeminfoReader) { - this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory; - this.timer = timer; - this.freezeTimeout = freezeTimeout; - this.spread = spread; - this.previousWantFrozen = true; - this.isFrozen = true; - this.startOfFreezeConvergence = timer.currentTime(); - - this.numberOfUnhandledExceptions = metrics.declareCounter("unhandled_exceptions", - new Dimensions(Map.of("src", "node-agents"))); - - this.procMeminfoReader = procMeminfoReader; - this.jvmHeapUsed = metrics.declareGauge(ContainerMetrics.MEM_HEAP_USED.baseName()); - this.jvmHeapFree = metrics.declareGauge(ContainerMetrics.MEM_HEAP_FREE.baseName()); - this.jvmHeapTotal = metrics.declareGauge(ContainerMetrics.MEM_HEAP_TOTAL.baseName()); - this.containerCount = metrics.declareGauge("container.count"); - this.metrics = metrics; - } - - @Override - public void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts) { - Map<String, NodeAgentContext> nodeAgentContextsByHostname = nodeAgentContexts.stream() - .collect(Collectors.toMap(ctx -> ctx.node().id(), Function.identity())); - - // Stop and remove NodeAgents that should no longer be running - diff(nodeAgentWithSchedulerByHostname.keySet(), nodeAgentContextsByHostname.keySet()) - .forEach(hostname -> nodeAgentWithSchedulerByHostname.remove(hostname).stopForRemoval()); - - // Start NodeAgent for hostnames that should be running, but aren't yet - diff(nodeAgentContextsByHostname.keySet(), nodeAgentWithSchedulerByHostname.keySet()).forEach(hostname -> { - NodeAgentWithScheduler naws = nodeAgentWithSchedulerFactory.create(nodeAgentContextsByHostname.get(hostname)); - naws.start(); - nodeAgentWithSchedulerByHostname.put(hostname, naws); - }); - - Duration timeBetweenNodeAgents = spread.dividedBy(Math.max(nodeAgentContextsByHostname.size() - 1, 1)); - Instant nextAgentStart = timer.currentTime(); - // At this point, nodeAgentContextsByHostname and nodeAgentWithSchedulerByHostname should have the same keys - for (Map.Entry<String, NodeAgentContext> entry : nodeAgentContextsByHostname.entrySet()) { - nodeAgentWithSchedulerByHostname.get(entry.getKey()).scheduleTickWith(entry.getValue(), nextAgentStart); - nextAgentStart = nextAgentStart.plus(timeBetweenNodeAgents); - } - } - - @Override - public void updateMetrics(boolean isSuspended) { - int numContainers = 0; - long totalContainerMemoryBytes = 0; - - for (NodeAgentWithScheduler nodeAgentWithScheduler : nodeAgentWithSchedulerByHostname.values()) { - int count = nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions(); - if (!isSuspended) numberOfUnhandledExceptions.add(count); - Optional<ContainerStats> containerStats = nodeAgentWithScheduler.updateContainerNodeMetrics(isSuspended); - if (containerStats.isPresent()) { - ++numContainers; - totalContainerMemoryBytes += containerStats.get().memoryStats().usage(); - } - } - - Runtime runtime = Runtime.getRuntime(); - runtime.gc(); - long freeMemory = runtime.freeMemory(); - long totalMemory = runtime.totalMemory(); - long usedMemory = totalMemory - freeMemory; - jvmHeapFree.sample(freeMemory); - jvmHeapUsed.sample(usedMemory); - jvmHeapTotal.sample(totalMemory); - - // No container stats are found while suspended, so skip setting these if so. - if (!isSuspended) { - containerCount.sample(numContainers); - ProcMeminfo meminfo = procMeminfoReader.read(); - updateMemoryOverheadMetric(numContainers, meminfo.memTotalBytes() - meminfo.memAvailableBytes() - totalContainerMemoryBytes); - } - } - - private void updateMemoryOverheadMetric(int numContainers, double memoryOverhead) { - final String name = "mem.system.overhead"; - Dimensions dimensions = new Dimensions(Map.of("containers", Integer.toString(numContainers))); - metrics.declareGauge(Metrics.APPLICATION_HOST, name, dimensions, Metrics.DimensionType.DEFAULT) - .sample(memoryOverhead); - if (previousMemoryOverheadDimensions != null && !previousMemoryOverheadDimensions.equals(dimensions)) - metrics.deleteMetricByDimension(name, previousMemoryOverheadDimensions, Metrics.DimensionType.DEFAULT); - previousMemoryOverheadDimensions = dimensions; - } - - @Override - public boolean setFrozen(boolean wantFrozen) { - if (wantFrozen != previousWantFrozen) { - if (wantFrozen) { - this.startOfFreezeConvergence = timer.currentTime(); - } else { - this.startOfFreezeConvergence = null; - } - - previousWantFrozen = wantFrozen; - } - - // Use filter with count instead of allMatch() because allMatch() will short circuit on first non-match - boolean allNodeAgentsConverged = parallelStreamOfNodeAgentWithScheduler() - .filter(nodeAgentScheduler -> !nodeAgentScheduler.setFrozen(wantFrozen, freezeTimeout)) - .count() == 0; - - if (wantFrozen) { - if (allNodeAgentsConverged) isFrozen = true; - } else isFrozen = false; - - return allNodeAgentsConverged; - } - - @Override - public boolean isFrozen() { - return isFrozen; - } - - @Override - public Duration subsystemFreezeDuration() { - if (startOfFreezeConvergence == null) { - return Duration.ZERO; - } else { - return Duration.between(startOfFreezeConvergence, timer.currentTime()); - } - } - - @Override - public void stopNodeAgentServices() { - // Each container may spend 1-1:30 minutes stopping - parallelStreamOfNodeAgentWithScheduler().forEach(NodeAgentWithScheduler::stopForHostSuspension); - } - - @Override - public void start() { - - } - - @Override - public void stop() { - // Stop all node-agents in parallel, will block until the last NodeAgent is stopped - parallelStreamOfNodeAgentWithScheduler().forEach(NodeAgentWithScheduler::stopForRemoval); - } - - /** - * Returns a parallel stream of NodeAgentWithScheduler. - * - * <p>Why not just call nodeAgentWithSchedulerByHostname.values().parallelStream()? Experiments - * with Java 11 have shown that with 10 nodes and forEach(), there are a maximum of 3 concurrent - * threads. With HashMap it produces 5. With List it produces 10 concurrent threads.</p> - */ - private Stream<NodeAgentWithScheduler> parallelStreamOfNodeAgentWithScheduler() { - return List.copyOf(nodeAgentWithSchedulerByHostname.values()).parallelStream(); - } - - // Set-difference. Returns minuend minus subtrahend. - private static <T> Set<T> diff(Set<T> minuend, Set<T> subtrahend) { - var result = new HashSet<>(minuend); - result.removeAll(subtrahend); - return result; - } - - static class NodeAgentWithScheduler implements NodeAgentScheduler { - private final NodeAgent nodeAgent; - private final NodeAgentScheduler nodeAgentScheduler; - - private NodeAgentWithScheduler(NodeAgent nodeAgent, NodeAgentScheduler nodeAgentScheduler) { - this.nodeAgent = nodeAgent; - this.nodeAgentScheduler = nodeAgentScheduler; - } - - void start() { nodeAgent.start(currentContext()); } - void stopForHostSuspension() { nodeAgent.stopForHostSuspension(currentContext()); } - void stopForRemoval() { nodeAgent.stopForRemoval(currentContext()); } - Optional<ContainerStats> updateContainerNodeMetrics(boolean isSuspended) { return nodeAgent.updateContainerNodeMetrics(currentContext(), isSuspended); } - int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); } - - @Override public void scheduleTickWith(NodeAgentContext context, Instant at) { nodeAgentScheduler.scheduleTickWith(context, at); } - @Override public boolean setFrozen(boolean frozen, Duration timeout) { return nodeAgentScheduler.setFrozen(frozen, timeout); } - @Override public NodeAgentContext currentContext() { return nodeAgentScheduler.currentContext(); } - } - - @FunctionalInterface - interface NodeAgentWithSchedulerFactory { - NodeAgentWithScheduler create(NodeAgentContext context); - } - - private static NodeAgentWithScheduler create(Timer timer, NodeAgentFactory nodeAgentFactory, NodeAgentContext context) { - NodeAgentContextManager contextManager = new NodeAgentContextManager(timer, context); - NodeAgent nodeAgent = nodeAgentFactory.create(contextManager, context); - return new NodeAgentWithScheduler(nodeAgent, contextManager); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java deleted file mode 100644 index dc10eaee46c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.concurrent.ThreadFactoryFactory; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; -import com.yahoo.yolean.Exceptions; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.RESUMED; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.TRANSITIONING; - -/** - * Pulls information from node repository and forwards containers to run to node admin. - * - * @author dybis, stiankri - */ -public class NodeAdminStateUpdater { - private static final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); - private static final Duration FREEZE_CONVERGENCE_TIMEOUT = Duration.ofMinutes(5); - - private final ScheduledExecutorService metricsScheduler = - Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler")); - - private final NodeAgentContextFactory nodeAgentContextFactory; - private final NodeRepository nodeRepository; - private final Orchestrator orchestrator; - private final NodeAdmin nodeAdmin; - private final String hostHostname; - - public enum State { TRANSITIONING, RESUMED, SUSPENDED_NODE_ADMIN, SUSPENDED } - - private volatile State currentState = SUSPENDED_NODE_ADMIN; - - public NodeAdminStateUpdater( - NodeAgentContextFactory nodeAgentContextFactory, - NodeRepository nodeRepository, - Orchestrator orchestrator, - NodeAdmin nodeAdmin, - HostName hostHostname) { - this.nodeAgentContextFactory = nodeAgentContextFactory; - this.nodeRepository = nodeRepository; - this.orchestrator = orchestrator; - this.nodeAdmin = nodeAdmin; - this.hostHostname = hostHostname.value(); - } - - public void start() { - nodeAdmin.start(); - - EnumSet<State> suspendedStates = EnumSet.of(SUSPENDED_NODE_ADMIN, SUSPENDED); - metricsScheduler.scheduleAtFixedRate(() -> { - try { - nodeAdmin.updateMetrics(suspendedStates.contains(currentState)); - } catch (Throwable e) { - log.log(Level.WARNING, "Metric fetcher scheduler failed", e); - } - }, 10, 55, TimeUnit.SECONDS); - } - - public void stop() { - metricsScheduler.shutdown(); - - // Stop all node-agents in parallel, will block until the last NodeAgent is stopped - nodeAdmin.stop(); - - do { - try { - metricsScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - } catch (InterruptedException e) { - log.info("Was interrupted while waiting for metricsScheduler and shutdown"); - } - } while (!metricsScheduler.isTerminated()); - } - - /** - * This method attempts to converge node-admin w/agents to a {@link State} - * with respect to: freeze, Orchestrator, and services running. - */ - public void converge(State wantedState) { - NodeSpec node = nodeRepository.getNode(hostHostname); - boolean hostIsActiveInNR = node.state() == NodeState.active; - if (wantedState == RESUMED) { - adjustNodeAgentsToRunFromNodeRepository(); - } else if (currentState == TRANSITIONING && nodeAdmin.subsystemFreezeDuration().compareTo(FREEZE_CONVERGENCE_TIMEOUT) > 0) { - // We have spent too much time trying to freeze and node admin is still not frozen. - // To avoid node agents stalling for too long, we'll force unfrozen ticks now. - adjustNodeAgentsToRunFromNodeRepository(); - nodeAdmin.setFrozen(false); - - if (hostIsActiveInNR) orchestrator.resume(hostHostname); - - throw ConvergenceException.ofTransient("Timed out trying to freeze all nodes: will force an unfrozen tick"); - } - - boolean wantFrozen = wantedState != RESUMED; - if (currentState == wantedState && wantFrozen == node.orchestratorStatus().isSuspended()) return; - currentState = TRANSITIONING; - - if (!nodeAdmin.setFrozen(wantFrozen)) - throw ConvergenceException.ofTransient("NodeAdmin is not yet " + (wantFrozen ? "frozen" : "unfrozen")); - - switch (wantedState) { - case RESUMED: - if (hostIsActiveInNR) orchestrator.resume(hostHostname); - break; - case SUSPENDED_NODE_ADMIN: - if (hostIsActiveInNR) orchestrator.suspend(hostHostname); - break; - case SUSPENDED: - // Fetch active nodes from node repo before suspending nodes. - // It is only possible to suspend active nodes, - // the orchestrator will fail if trying to suspend nodes in other states. - // Even though state is frozen we need to interact with node repo, but - // the data from node repo should not be used for anything else. - // We should also suspend host's hostname to suspend node-admin - List<String> nodesInActiveState = getNodesInActiveState(); - - List<String> nodesToSuspend = new ArrayList<>(nodesInActiveState); - if (hostIsActiveInNR) nodesToSuspend.add(hostHostname); - if (!nodesToSuspend.isEmpty()) { - orchestrator.suspend(hostHostname, nodesToSuspend); - log.info("Orchestrator allows suspension of " + nodesToSuspend); - } - - // The node agent services are stopped by this thread, which is OK only - // because the node agents are frozen (see above). - nodeAdmin.stopNodeAgentServices(); - break; - default: - throw new IllegalStateException("Unknown wanted state " + wantedState); - } - - log.info("State changed from " + currentState + " to " + wantedState); - currentState = wantedState; - } - - void adjustNodeAgentsToRunFromNodeRepository() { - try { - Map<String, Acl> aclByHostname = nodeRepository.getAcls(hostHostname); - - Set<NodeAgentContext> nodeAgentContexts = nodeRepository.getNodes(hostHostname).stream() - .map(node -> nodeAgentContextFactory.create(node, aclByHostname.getOrDefault(node.hostname(), Acl.EMPTY))) - .collect(Collectors.toSet()); - nodeAdmin.refreshContainersToRun(nodeAgentContexts); - } catch (ConvergenceException e) { - log.log(Level.WARNING, "Failed to update which containers should be running: " + Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to update which containers should be running", e); - } - } - - private List<String> getNodesInActiveState() { - return nodeRepository.getNodes(hostHostname) - .stream() - .filter(node -> node.state() == NodeState.active) - .map(NodeSpec::hostname) - .toList(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfo.java deleted file mode 100644 index 0c0d8dc348c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfo.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -/** - * Represents /proc/meminfo, see proc(5). - * - * @param memTotalBytes Total usable RAM (i.e., physical RAM minus a few reserved bits and the kernel binary code). - * @param memAvailableBytes An estimate of how much memory is available for starting new applications, without swapping. - * - * @author hakon - */ -public record ProcMeminfo(long memTotalBytes, long memAvailableBytes) { } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfoReader.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfoReader.java deleted file mode 100644 index d13aa1ea03c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/ProcMeminfoReader.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.yolean.Exceptions; - -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Reads /proc/meminfo, see proc(5). - * - * @author hakon - */ -public class ProcMeminfoReader { - private static final String PROC_MEMINFO = "/proc/meminfo"; - private static final Pattern MEM_TOTAL_PATTERN = Pattern.compile("MemTotal: *([0-9]+) kB"); - private static final Pattern MEM_AVAILABLE_PATTERN = Pattern.compile("MemAvailable: *([0-9]+) kB"); - - private final FileSystem fileSystem; - - public ProcMeminfoReader(FileSystem fileSystem) { - this.fileSystem = fileSystem; - } - - public ProcMeminfo read() { - return read(Exceptions.uncheck(() -> Files.readString(fileSystem.getPath(PROC_MEMINFO)))); - } - - static ProcMeminfo read(String meminfoContent) { - return new ProcMeminfo(readKbGroup(meminfoContent, MEM_TOTAL_PATTERN), - readKbGroup(meminfoContent, MEM_AVAILABLE_PATTERN)); - } - - private static long readKbGroup(String string, Pattern pattern) { - Matcher matcher = pattern.matcher(string); - if (!matcher.find()) - throw new IllegalArgumentException(pattern + " did not match anything in " + PROC_MEMINFO); - return Long.parseLong(matcher.group(1)) * 1024; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java deleted file mode 100644 index 68af4e59d45..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java deleted file mode 100644 index 3f7ff63c90b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import java.nio.file.Path; - -/** - * Utility for manipulating the initial file system the Docker container will start with. - * - * @author hakon - */ -public interface ContainerData { - - /** Add or overwrite file in container at path. */ - void addFile(ContainerPath path, String data); - - /** - * @param path Container path to write - * @param data UTF-8 file content - * @param permissions file permissions, see {@link UnixPath#setPermissions(String)} for format. - */ - void addFile(ContainerPath path, String data, String permissions); - - /** - * @param path Container path to create directory at - * @param permissions optional file permissions, see {@link UnixPath#setPermissions(String)} for format. - */ - void addDirectory(ContainerPath path, String... permissions); - - /** - * Symlink to a file in container at path. - * @param symlink The path to the symlink inside the container - * @param target The path to the target file for the symbolic link inside the container - */ - void addSymlink(ContainerPath symlink, Path target); - - /** Writes all the files, directories and symlinks that were previously added */ - void converge(NodeAgentContext context); -} - diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java deleted file mode 100644 index 78c907ad277..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/HealthChecker.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -/** - * Interface for verifying the health of the node. - * - * @author hakonhall - */ -public interface HealthChecker extends AutoCloseable { - /** Verify the health of an active node, just before updating the node repo and calling Orchestrator resume. */ - void verifyHealth(NodeAgentContext context); - - @Override - void close(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java deleted file mode 100644 index b37b4dd665a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.vespa.hosted.node.admin.container.ContainerStats; - -import java.util.Optional; - -/** - * Responsible for management of a single node over its lifecycle. - * May own its own resources, threads etc. Runs independently, but receives signals - * on state changes in the environment that may trigger this agent to take actions. - * - * @author bakksjo - */ -public interface NodeAgent { - /** - * Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously - * striving to make the current state equal to the wanted state. - */ - void start(NodeAgentContext context); - - /** - * Stop the node in anticipation of host suspension, e.g. reboot or docker upgrade. - */ - void stopForHostSuspension(NodeAgentContext context); - - /** - * Signals to the agent that the node is at the end of its lifecycle and no longer needs a managing agent. - * Cleans up any resources the agent owns, such as threads, connections etc. Cleanup is synchronous; when this - * method returns, no more actions will be taken by the agent. - */ - void stopForRemoval(NodeAgentContext context); - - /** - * Updates metric receiver with the latest node-agent stats, and returns the container stats if available. - */ - default Optional<ContainerStats> updateContainerNodeMetrics(NodeAgentContext context, boolean isSuspended) { return Optional.empty(); } - - /** - * Returns and resets number of unhandled exceptions - */ - int getAndResetNumberOfUnhandledExceptions(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java deleted file mode 100644 index 9409ae2bee1..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.container.ContainerNetworkMode; - -import java.util.Optional; - -public interface NodeAgentContext extends TaskContext { - - /** @return node specification from node-repository */ - NodeSpec node(); - - /** @return node ACL from node-repository */ - Acl acl(); - - /** @return name of the linux container this context applies to */ - ContainerName containerName(); - - /** @return hostname of the linux container this context applies to */ - default HostName hostname() { - return HostName.of(node().hostname()); - } - - default NodeType nodeType() { - return node().type(); - } - - AthenzIdentity identity(); - - ContainerNetworkMode networkMode(); - - ZoneApi zone(); - - /** @return information about users/user namespace of the linux container this context applies to */ - UserScope users(); - - /** @return methods to resolve paths within container's file system */ - PathScope paths(); - - default boolean isDisabled(NodeAgentTask task) { - return false; - } - - /** - * The vcpu value in NodeSpec is the number of vcpus required by the node on a fixed historical - * baseline machine. However the current host has a faster per-vcpu performance by a scale factor - * (see flavors.def cpuSpeedup), and therefore do not need to set aside the full number of vcpus - * to run the node. This method returns that reduced number of vcpus. - * - * @return the vcpus required by the node on this host. - */ - double vcpuOnThisHost(); - - Optional<ApplicationId> hostExclusiveTo(); - - boolean exclave(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java deleted file mode 100644 index 4e8db239867..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; - -/** - * @author freva - */ -@FunctionalInterface -public interface NodeAgentContextFactory { - NodeAgentContext create(NodeSpec nodeSpec, Acl acl); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java deleted file mode 100644 index 21d1cfd632c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.container.ContainerNetworkMode; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystem; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author freva - */ -public class NodeAgentContextImpl implements NodeAgentContext { - - private final String logPrefix; - private final NodeSpec node; - private final Acl acl; - private final ContainerName containerName; - private final AthenzIdentity identity; - private final ContainerNetworkMode containerNetworkMode; - private final ZoneApi zone; - private final UserScope userScope; - private final PathScope pathScope; - private final double cpuSpeedup; - private final Set<NodeAgentTask> disabledNodeAgentTasks; - private final Optional<ApplicationId> hostExclusiveTo; - private final boolean exclave; - - public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity, - ContainerNetworkMode containerNetworkMode, ZoneApi zone, - FlagSource flagSource, UserScope userScope, PathScope pathScope, - double cpuSpeedup, Optional<ApplicationId> hostExclusiveTo, boolean exclave) { - if (cpuSpeedup <= 0) - throw new IllegalArgumentException("cpuSpeedUp must be positive, was: " + cpuSpeedup); - - this.node = Objects.requireNonNull(node); - this.acl = Objects.requireNonNull(acl); - this.containerName = ContainerName.fromHostname(node.hostname()); - this.identity = Objects.requireNonNull(identity); - this.containerNetworkMode = Objects.requireNonNull(containerNetworkMode); - this.zone = Objects.requireNonNull(zone); - this.userScope = Objects.requireNonNull(userScope); - this.pathScope = Objects.requireNonNull(pathScope); - this.logPrefix = containerName.asString() + ": "; - this.cpuSpeedup = cpuSpeedup; - this.disabledNodeAgentTasks = NodeAgentTask.fromString( - PermanentFlags.DISABLED_HOST_ADMIN_TASKS.bindTo(flagSource) - .with(FetchVector.Dimension.HOSTNAME, node.hostname()) - .with(FetchVector.Dimension.NODE_TYPE, node.type().name()).value()); - this.hostExclusiveTo = hostExclusiveTo; - this.exclave = exclave; - } - - @Override - public NodeSpec node() { - return node; - } - - @Override - public Acl acl() { - return acl; - } - - @Override - public ContainerName containerName() { - return containerName; - } - - @Override - public AthenzIdentity identity() { - return identity; - } - - @Override - public ContainerNetworkMode networkMode() { - return containerNetworkMode; - } - - @Override - public ZoneApi zone() { - return zone; - } - - @Override - public UserScope users() { - return userScope; - } - - @Override - public PathScope paths() { - return pathScope; - } - - @Override - public boolean isDisabled(NodeAgentTask task) { - return disabledNodeAgentTasks.contains(task); - } - - @Override - public double vcpuOnThisHost() { - return node.vcpu() / cpuSpeedup; - } - - @Override - public Optional<ApplicationId> hostExclusiveTo() { - return hostExclusiveTo; - } - - @Override - public void recordSystemModification(Logger logger, String message) { - log(logger, message); - } - - @Override - public void log(Logger logger, Level level, String message) { - logger.log(level, logPrefix + message); - } - - @Override - public void log(Logger logger, Level level, String message, Throwable throwable) { - logger.log(level, logPrefix + message, throwable); - } - - @Override - public boolean exclave() { - return exclave; - } - - public static NodeAgentContextImpl.Builder builder(NodeSpec node) { - return new Builder(new NodeSpec.Builder(node)); - } - - /** - * Creates a NodeAgentContext.Builder with a NodeSpec that has the given hostname and some - * reasonable values for the remaining required NodeSpec fields. Use {@link #builder(NodeSpec)} - * if you want to control the entire NodeSpec. - */ - public static NodeAgentContextImpl.Builder builder(String hostname) { - return new Builder(NodeSpec.Builder.testSpec(hostname)); - } - - /** For testing only! */ - public static class Builder { - private static final Path DEFAULT_CONTAINER_STORAGE = Path.of("/data/vespa/storage"); - - private NodeSpec.Builder nodeSpecBuilder; - private Acl acl; - private AthenzIdentity identity; - private ContainerNetworkMode containerNetworkMode; - private ZoneApi zone; - private UserNamespace userNamespace; - private Path containerStorage; - private FlagSource flagSource; - private double cpuSpeedUp = 1; - private Optional<ApplicationId> hostExclusiveTo = Optional.empty(); - private boolean exclave = false; - - private Builder(NodeSpec.Builder nodeSpecBuilder) { - this.nodeSpecBuilder = nodeSpecBuilder; - } - - public Builder nodeSpecBuilder(Function<NodeSpec.Builder, NodeSpec.Builder> nodeSpecBuilderModifier) { - this.nodeSpecBuilder = nodeSpecBuilderModifier.apply(nodeSpecBuilder); - return this; - } - - public Builder acl(Acl acl) { - this.acl = acl; - return this; - } - - public Builder identity(AthenzIdentity identity) { - this.identity = identity; - return this; - } - - public Builder networkMode(ContainerNetworkMode containerNetworkMode) { - this.containerNetworkMode = containerNetworkMode; - return this; - } - - public Builder zone(ZoneApi zone) { - this.zone = zone; - return this; - } - - public Builder userNamespace(UserNamespace userNamespace) { - this.userNamespace = userNamespace; - return this; - } - - /** Sets the file system to use for paths. */ - public Builder fileSystem(FileSystem fileSystem) { - return containerStorage(fileSystem.getPath(DEFAULT_CONTAINER_STORAGE.toString())); - } - - public Builder flagSource(FlagSource flagSource) { - this.flagSource = flagSource; - return this; - } - - public Builder cpuSpeedUp(double cpuSpeedUp) { - this.cpuSpeedUp = cpuSpeedUp; - return this; - } - - public Builder containerStorage(Path path) { - this.containerStorage = path; - return this; - } - - public Builder hostExclusiveTo(ApplicationId applicationId) { - this.hostExclusiveTo = Optional.ofNullable(applicationId); - return this; - } - - public Builder exclave(boolean exclave) { - this.exclave = exclave; - return this; - } - - public NodeAgentContextImpl build() { - Objects.requireNonNull(containerStorage, "Must set one of containerStorage or fileSystem"); - - UserScope userScope = UserScope.create( - Optional.ofNullable(userNamespace).orElseGet(() -> new UserNamespace(100000, 100000, 100000))); - ContainerFileSystem containerFs = ContainerFileSystem.create(containerStorage - .resolve(nodeSpecBuilder.hostname().split("\\.")[0]), userScope); - containerFs.createRoot(); - - return new NodeAgentContextImpl( - nodeSpecBuilder.build(), - Optional.ofNullable(acl).orElse(Acl.EMPTY), - Optional.ofNullable(identity).orElseGet(() -> new AthenzService("domain", "service")), - Optional.ofNullable(containerNetworkMode).orElse(ContainerNetworkMode.HOST_NETWORK), - Optional.ofNullable(zone).orElseGet(() -> new ZoneApi() { - @Override - public SystemName getSystemName() { - return SystemName.defaultSystem(); - } - - @Override - public ZoneId getId() { - return ZoneId.defaultId(); - } - - @Override - public CloudName getCloudName() { - return CloudName.DEFAULT; - } - - @Override - public String getCloudNativeRegionName() { - return getId().region().value(); - } - }), - Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new), - userScope, - new PathScope(containerFs, "/opt/vespa"), - cpuSpeedUp, hostExclusiveTo, exclave); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java deleted file mode 100644 index ee3c86b838f..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.jdisc.Timer; - -import java.time.Duration; -import java.time.Instant; -import java.util.Objects; - -/** - * This class should be used by exactly 2 threads, 1 for each interface it implements. - * - * @author freva - */ -public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAgentScheduler { - - private final Object monitor = new Object(); - private final Timer timer; - - private NodeAgentContext currentContext; - private NodeAgentContext nextContext; - private Instant nextContextAt; - private boolean wantFrozen = false; - private boolean isFrozen = true; - private boolean interrupted = false; - private boolean isWaitingForNextContext = false; - - public NodeAgentContextManager(Timer timer, NodeAgentContext context) { - this.timer = timer; - this.currentContext = context; - } - - @Override - public void scheduleTickWith(NodeAgentContext context, Instant at) { - synchronized (monitor) { - nextContext = Objects.requireNonNull(context); - nextContextAt = Objects.requireNonNull(at); - monitor.notifyAll(); // Notify of new context - } - } - - @Override - public boolean setFrozen(boolean frozen, Duration timeout) { - synchronized (monitor) { - if (wantFrozen != frozen) { - wantFrozen = frozen; - monitor.notifyAll(); // Notify the supplier of the wantFrozen change - } - - boolean successful; - long remainder; - long end = timer.currentTime().plus(timeout).toEpochMilli(); - while (!(successful = isFrozen == frozen) && (remainder = end - timer.currentTimeMillis()) > 0) { - try { - monitor.wait(remainder); // Wait with timeout until the supplier is has reached wanted frozen state - } catch (InterruptedException ignored) { } - } - - return successful; - } - } - - @Override - public NodeAgentContext nextContext() throws ContextSupplierInterruptedException { - synchronized (monitor) { - nextContext = null; // Reset any previous context and wait for the next one - isWaitingForNextContext = true; - monitor.notifyAll(); - Duration untilNextContext = Duration.ZERO; - while (true) { - if (interrupted) throw new ContextSupplierInterruptedException(); - - if (!setAndGetIsFrozen(wantFrozen) && - nextContext != null && - (untilNextContext = Duration.between(Instant.now(), nextContextAt)).toMillis() <= 0) - break; - - try { - monitor.wait(Math.max(untilNextContext.toMillis(), 0L)); // Wait until scheduler provides a new context - } catch (InterruptedException ignored) { } - } - - isWaitingForNextContext = false; - currentContext = nextContext; - return currentContext; - } - } - - @Override - public NodeAgentContext currentContext() { - synchronized (monitor) { - return currentContext; - } - } - - @Override - public void interrupt() { - synchronized (monitor) { - interrupted = true; - monitor.notifyAll(); - } - } - - private boolean setAndGetIsFrozen(boolean isFrozen) { - synchronized (monitor) { - if (this.isFrozen != isFrozen) { - this.isFrozen = isFrozen; - monitor.notifyAll(); // Notify the scheduler of the isFrozen change - } - return this.isFrozen; - } - } - - /** FOR TESTING ONLY */ - void waitUntilWaitingForNextContext() { - synchronized (monitor) { - while (!isWaitingForNextContext) { - try { - monitor.wait(); - } catch (InterruptedException ignored) { } - } - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java deleted file mode 100644 index a4450626766..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -/** - * @author freva - */ -public interface NodeAgentContextSupplier { - - /** - * Blocks until the next context is ready - * @return context - * @throws ContextSupplierInterruptedException if {@link #interrupt()} was called before this method returned - */ - NodeAgentContext nextContext() throws ContextSupplierInterruptedException; - - /** Interrupts the thread(s) currently waiting in {@link #nextContext()} */ - void interrupt(); - - class ContextSupplierInterruptedException extends RuntimeException { } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java deleted file mode 100644 index ef67ff88471..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -/** - * @author freva - */ -@FunctionalInterface -public interface NodeAgentFactory { - NodeAgent create(NodeAgentContextSupplier contextSupplier, NodeAgentContext nodeAgentContext); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java deleted file mode 100644 index 43dc3d72c46..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ /dev/null @@ -1,633 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.flags.DoubleFlag; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.container.Container; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentialsProvider; -import com.yahoo.vespa.hosted.node.admin.maintenance.ContainerWireguardTask; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; -import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; -import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextSupplier.ContextSupplierInterruptedException; -import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.ABSENT; -import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.STARTING; -import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.UNKNOWN; - -/** - * @author dybis - * @author bakksjo - */ -public class NodeAgentImpl implements NodeAgent { - - // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration - // Subtract 1 second to avoid warmup coming in lockstep with tick time and always end up using an extra tick when there are just a few ms left - private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(90).minus(Duration.ofSeconds(1)); - - private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName()); - - private final NodeAgentContextSupplier contextSupplier; - private final NodeRepository nodeRepository; - private final Orchestrator orchestrator; - private final ContainerOperations containerOperations; - private final RegistryCredentialsProvider registryCredentialsProvider; - private final StorageMaintainer storageMaintainer; - private final List<CredentialsMaintainer> credentialsMaintainers; - private final Optional<AclMaintainer> aclMaintainer; - private final Optional<HealthChecker> healthChecker; - private final Timer timer; - private final Duration warmUpDuration; - private final DoubleFlag containerCpuCap; - private final VespaServiceDumper serviceDumper; - private final List<ContainerWireguardTask> wireguardTasks; - - private Thread loopThread; - private ContainerState containerState = UNKNOWN; - private NodeSpec lastNode; - - private final AtomicBoolean terminated = new AtomicBoolean(false); - private boolean hasResumedNode = false; - private boolean hasStartedServices = true; - private Optional<Instant> firstSuccessfulHealthCheckInstant = Optional.empty(); - private boolean suspendedInOrchestrator = false; - - private int numberOfUnhandledException = 0; - private long currentRebootGeneration = 0; - private Optional<Long> currentRestartGeneration = Optional.empty(); - - /** - * ABSENT means container is definitely absent - A container that was absent will not suddenly appear without - * NodeAgent explicitly starting it. - * STARTING state is set just before we attempt to start a container, if successful we move to the next state. - * Otherwise we can't be certain. A container that was running a minute ago may no longer be running without - * NodeAgent doing anything (container could have crashed). Therefore we always have to ask docker daemon - * to get updated state of the container. - */ - enum ContainerState { - ABSENT, - STARTING, - UNKNOWN - } - - - public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository, - Orchestrator orchestrator, ContainerOperations containerOperations, - RegistryCredentialsProvider registryCredentialsProvider, StorageMaintainer storageMaintainer, - FlagSource flagSource, List<CredentialsMaintainer> credentialsMaintainers, - Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Timer timer, - VespaServiceDumper serviceDumper, List<ContainerWireguardTask> wireguardTasks) { - this(contextSupplier, nodeRepository, orchestrator, containerOperations, registryCredentialsProvider, - storageMaintainer, flagSource, credentialsMaintainers, aclMaintainer, healthChecker, timer, - DEFAULT_WARM_UP_DURATION, serviceDumper, wireguardTasks); - } - - public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository, - Orchestrator orchestrator, ContainerOperations containerOperations, - RegistryCredentialsProvider registryCredentialsProvider, StorageMaintainer storageMaintainer, - FlagSource flagSource, List<CredentialsMaintainer> credentialsMaintainers, - Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Timer timer, - Duration warmUpDuration, VespaServiceDumper serviceDumper, - List<ContainerWireguardTask> wireguardTasks) { - this.contextSupplier = contextSupplier; - this.nodeRepository = nodeRepository; - this.orchestrator = orchestrator; - this.containerOperations = containerOperations; - this.registryCredentialsProvider = registryCredentialsProvider; - this.storageMaintainer = storageMaintainer; - this.credentialsMaintainers = credentialsMaintainers; - this.aclMaintainer = aclMaintainer; - this.healthChecker = healthChecker; - this.timer = timer; - this.warmUpDuration = warmUpDuration; - this.containerCpuCap = PermanentFlags.CONTAINER_CPU_CAP.bindTo(flagSource); - this.serviceDumper = serviceDumper; - this.wireguardTasks = new ArrayList<>(wireguardTasks); - } - - @Override - public void start(NodeAgentContext initialContext) { - if (loopThread != null) - throw new IllegalStateException("Can not re-start a node agent."); - - loopThread = new Thread(() -> { - while (!terminated.get()) { - try { - converge(contextSupplier.nextContext()); - } catch (ContextSupplierInterruptedException ignored) { } - } - }); - loopThread.setName("tick-" + initialContext.hostname()); - loopThread.start(); - } - - @Override - public void stopForRemoval(NodeAgentContext context) { - if (!terminated.compareAndSet(false, true)) - throw new IllegalStateException("Can not re-stop a node agent."); - - contextSupplier.interrupt(); - - do { - try { - loopThread.join(); - } catch (InterruptedException ignored) { } - } while (loopThread.isAlive()); - - context.log(logger, "Stopped"); - } - - void startServicesIfNeeded(NodeAgentContext context) { - if (!hasStartedServices) { - context.log(logger, "Invoking vespa-nodectl to start services"); - String output = containerOperations.startServices(context); - if (!output.isBlank()) { - context.log(logger, "Start services output: " + output); - } - hasStartedServices = true; - } - } - - void resumeNodeIfNeeded(NodeAgentContext context) { - if (!hasResumedNode) { - context.log(logger, "Invoking vespa-nodectl to resume services"); - String output = containerOperations.resumeNode(context); - if (!output.isBlank()) { - context.log(logger, "Resume services output: " + output); - } - hasResumedNode = true; - } - } - - private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context, Optional<Instant> containerCreatedAt) { - final NodeAttributes currentNodeAttributes = new NodeAttributes(); - final NodeAttributes newNodeAttributes = new NodeAttributes(); - boolean changed = false; - - if (context.node().wantedRestartGeneration().isPresent() && - !Objects.equals(context.node().currentRestartGeneration(), currentRestartGeneration)) { - currentNodeAttributes.withRestartGeneration(context.node().currentRestartGeneration()); - newNodeAttributes.withRestartGeneration(currentRestartGeneration); - changed = true; - } - - boolean createdAtAfterRebootedEvent = context.node().events().stream() - .filter(event -> event.type().equals("rebooted")) - .map(event -> containerCreatedAt - .map(createdAt -> createdAt.isAfter(event.at())) - .orElse(false)) // Container not created - .findFirst() - .orElse(containerCreatedAt.isPresent()); // No rebooted event - if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration) || createdAtAfterRebootedEvent) { - currentNodeAttributes.withRebootGeneration(context.node().currentRebootGeneration()); - newNodeAttributes.withRebootGeneration(currentRebootGeneration); - changed = true; - } - - Optional<DockerImage> wantedDockerImage = context.node().wantedDockerImage().filter(n -> containerState == UNKNOWN); - if (!Objects.equals(context.node().currentDockerImage(), wantedDockerImage)) { - DockerImage currentImage = context.node().currentDockerImage().orElse(DockerImage.EMPTY); - DockerImage newImage = wantedDockerImage.orElse(DockerImage.EMPTY); - - currentNodeAttributes.withDockerImage(currentImage); - currentNodeAttributes.withVespaVersion(context.node().currentVespaVersion().orElse(Version.emptyVersion)); - newNodeAttributes.withDockerImage(newImage); - newNodeAttributes.withVespaVersion(context.node().wantedVespaVersion().orElse(Version.emptyVersion)); - changed = true; - } - - Optional<DropDocumentsReport> report = context.node().reports().getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class); - if (report.isPresent() && report.get().startedAt() == null && report.get().readiedAt() != null) { - newNodeAttributes.withReport(DropDocumentsReport.reportId(), report.get().withStartedAt(timer.currentTimeMillis()).toJsonNode()); - changed = true; - } - - if (changed) { - context.log(logger, "Publishing new set of attributes to node repo: %s -> %s", - currentNodeAttributes, newNodeAttributes); - nodeRepository.updateNodeAttributes(context.hostname().value(), newNodeAttributes); - } - } - - private Container startContainer(NodeAgentContext context) { - ContainerResources wantedResources = warmUpDuration(context).isNegative() ? - getContainerResources(context) : getContainerResources(context).withUnlimitedCpus(); - ContainerData containerData = containerOperations.createContainer(context, wantedResources); - writeContainerData(context, containerData); - containerOperations.startContainer(context); - - currentRebootGeneration = context.node().wantedRebootGeneration(); - currentRestartGeneration = context.node().wantedRestartGeneration(); - hasStartedServices = true; // Automatically started with the container - hasResumedNode = false; - context.log(logger, "Container successfully started, new containerState is " + containerState); - return containerOperations.getContainer(context).orElseThrow(() -> - ConvergenceException.ofError("Did not find container that was just started")); - } - - private Optional<Container> removeContainerIfNeededUpdateContainerState( - NodeAgentContext context, Optional<Container> existingContainer) { - if (existingContainer.isPresent()) { - List<String> reasons = shouldRemoveContainer(context, existingContainer.get()); - if (!reasons.isEmpty()) { - removeContainer(context, existingContainer.get(), reasons, false); - return Optional.empty(); - } - - shouldRestartServices(context, existingContainer.get()).ifPresent(restartReason -> { - context.log(logger, "Invoking vespa-nodectl to restart services: " + restartReason); - orchestratorSuspendNode(context); - - ContainerResources currentResources = existingContainer.get().resources(); - ContainerResources wantedResources = currentResources.withUnlimitedCpus(); - if ( ! warmUpDuration(context).isNegative() && ! wantedResources.equals(currentResources)) { - context.log(logger, "Updating container resources: %s -> %s", - existingContainer.get().resources().toStringCpu(), wantedResources.toStringCpu()); - containerOperations.updateContainer(context, existingContainer.get().id(), wantedResources); - } - - String output = containerOperations.restartVespa(context); - if ( ! output.isBlank()) { - context.log(logger, "Restart services output: " + output); - } - currentRestartGeneration = context.node().wantedRestartGeneration(); - firstSuccessfulHealthCheckInstant = Optional.empty(); - }); - } - - return existingContainer; - } - - private Optional<String> shouldRestartServices(NodeAgentContext context, Container existingContainer) { - NodeSpec node = context.node(); - if (!existingContainer.state().isRunning() || node.state() != NodeState.active) return Optional.empty(); - - // Restart generation is only optional because it does not exist for unallocated nodes - if (currentRestartGeneration.get() < node.wantedRestartGeneration().get()) { - return Optional.of("Restart requested - wanted restart generation has been bumped: " - + currentRestartGeneration.get() + " -> " + node.wantedRestartGeneration().get()); - } - - return Optional.empty(); - } - - private void stopServicesIfNeeded(NodeAgentContext context) { - if (hasStartedServices && context.node().owner().isEmpty()) - stopServices(context); - } - - private void stopServices(NodeAgentContext context) { - context.log(logger, "Stopping services"); - if (containerState == ABSENT) return; - hasStartedServices = hasResumedNode = false; - firstSuccessfulHealthCheckInstant = Optional.empty(); - containerOperations.stopServices(context); - } - - @Override - public void stopForHostSuspension(NodeAgentContext context) { - getContainer(context).ifPresent(container -> removeContainer(context, container, List.of("Suspending host"), true)); - } - - public void suspend(NodeAgentContext context) { - if (containerState == ABSENT) return; - try { - hasResumedNode = false; - context.log(logger, "Invoking vespa-nodectl to suspend services"); - String output = containerOperations.suspendNode(context); - if (!output.isBlank()) { - context.log(logger, "Suspend services output: " + output); - } - } catch (RuntimeException e) { - // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to - // remove container, we will not be able to upgrade to fix any problems in the suspend logic! - context.log(logger, Level.WARNING, "Failed trying to suspend container", e); - } - } - - private List<String> shouldRemoveContainer(NodeAgentContext context, Container existingContainer) { - final NodeState nodeState = context.node().state(); - List<String> reasons = new ArrayList<>(); - if (nodeState == NodeState.dirty || nodeState == NodeState.provisioned) - reasons.add("Node in state " + nodeState + ", container should no longer be running"); - - if (context.node().wantedDockerImage().isPresent() && - !context.node().wantedDockerImage().get().equals(existingContainer.image())) { - reasons.add("The node is supposed to run a new Docker image: " - + existingContainer.image().asString() + " -> " + context.node().wantedDockerImage().get().asString()); - } - - if (!existingContainer.state().isRunning()) - reasons.add("Container no longer running"); - - if (currentRebootGeneration < context.node().wantedRebootGeneration()) { - reasons.add(String.format("Container reboot wanted. Current: %d, Wanted: %d", - currentRebootGeneration, context.node().wantedRebootGeneration())); - } - - ContainerResources wantedContainerResources = getContainerResources(context); - if (!wantedContainerResources.equalsMemory(existingContainer.resources())) { - reasons.add("Container should be running with different memory allocation, wanted: " + - wantedContainerResources.toStringMemory() + ", actual: " + existingContainer.resources().toStringMemory()); - } - - if (containerState == STARTING) - reasons.add("Container failed to start"); - - return reasons; - } - - private void removeContainer(NodeAgentContext context, Container existingContainer, List<String> reasons, boolean alreadySuspended) { - context.log(logger, "Will remove container: " + String.join(", ", reasons)); - - if (existingContainer.state().isRunning()) { - if (!alreadySuspended) { - orchestratorSuspendNode(context); - } - - try { - if (context.node().state() == NodeState.active) { - suspend(context); - } - stopServices(context); - } catch (Exception e) { - context.log(logger, Level.WARNING, "Failed stopping services, ignoring", e); - } - } - - storageMaintainer.handleCoreDumpsForContainer(context, Optional.of(existingContainer), true); - containerOperations.removeContainer(context, existingContainer); - containerState = ABSENT; - context.log(logger, "Container successfully removed, new containerState is " + containerState); - } - - - private Container updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) { - ContainerResources wantedContainerResources = getContainerResources(context); - - if (healthChecker.isPresent() && firstSuccessfulHealthCheckInstant - .map(timer.currentTime().minus(warmUpDuration(context))::isBefore) - .orElse(true)) - return existingContainer; - - if (wantedContainerResources.equalsCpu(existingContainer.resources())) return existingContainer; - context.log(logger, "Container should be running with different CPU allocation, wanted: %s, current: %s", - wantedContainerResources.toStringCpu(), existingContainer.resources().toStringCpu()); - - // Only update CPU resources - containerOperations.updateContainer(context, existingContainer.id(), wantedContainerResources.withMemoryBytes(existingContainer.resources().memoryBytes())); - return containerOperations.getContainer(context).orElseThrow(() -> - ConvergenceException.ofError("Did not find container that was just updated")); - } - - private ContainerResources getContainerResources(NodeAgentContext context) { - double cpuCap = context.vcpuOnThisHost() * containerCpuCap - .with(INSTANCE_ID, context.node().owner().map(ApplicationId::serializedForm)) - .with(CLUSTER_ID, context.node().membership().map(NodeMembership::clusterId)) - .with(CLUSTER_TYPE, context.node().membership().map(membership -> membership.type().value())) - .with(HOSTNAME, context.node().hostname()) - .value(); - - return ContainerResources.from(cpuCap, context.vcpuOnThisHost(), context.node().memoryGb()); - } - - private boolean downloadImageIfNeeded(NodeAgentContext context, Optional<Container> container) { - NodeSpec node = context.node(); - if (node.wantedDockerImage().equals(container.map(c -> c.image()))) return false; - - return node.wantedDockerImage() - .map(image -> containerOperations.pullImageAsyncIfNeeded(context, image, registryCredentialsProvider)) - .orElse(false); - } - - private void dropDocsIfNeeded(NodeAgentContext context, Optional<Container> container) { - Optional<DropDocumentsReport> report = context.node().reports() - .getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class); - if (report.isEmpty() || report.get().readiedAt() != null) return; - - if (report.get().droppedAt() == null) { - container.ifPresent(c -> removeContainer(context, c, List.of("Dropping documents"), true)); - FileFinder.from(context.paths().underVespaHome("var/db/vespa/search")).deleteRecursively(context); - nodeRepository.updateNodeAttributes(context.node().hostname(), - new NodeAttributes().withReport(DropDocumentsReport.reportId(), report.get().withDroppedAt(timer.currentTimeMillis()).toJsonNode())); - } - - throw ConvergenceException.ofTransient("Documents already dropped, waiting for signal to start the container"); - } - - public void converge(NodeAgentContext context) { - try { - doConverge(context); - context.log(logger, Level.INFO, "Converged"); - } catch (ConvergenceException e) { - context.log(logger, e.getMessage()); - if (e.isError()) - numberOfUnhandledException++; - } catch (Throwable e) { - numberOfUnhandledException++; - context.log(logger, Level.SEVERE, "Unhandled exception, ignoring", e); - } - } - - // Non-private for testing - void doConverge(NodeAgentContext context) { - NodeSpec node = context.node(); - Optional<Container> container = getContainer(context); - - // Current reboot generation uninitialized or incremented from outside to cancel reboot - if (currentRebootGeneration < node.currentRebootGeneration()) - currentRebootGeneration = node.currentRebootGeneration(); - - // Either we have changed allocation status (restart gen. only available to allocated nodes), or - // restart generation has been incremented from outside to cancel restart - if (currentRestartGeneration.isPresent() != node.currentRestartGeneration().isPresent() || - currentRestartGeneration.map(current -> current < node.currentRestartGeneration().get()).orElse(false)) - currentRestartGeneration = node.currentRestartGeneration(); - - if (!node.equals(lastNode)) { - logChangesToNodeSpec(context, lastNode, node); - lastNode = node; - } - - // Run this here and now, even though we may immediately remove the container below. - // This ensures these maintainers are run even if something fails or returns early. - // These maintainers should also run immediately after starting the container (see below). - container.filter(c -> c.state().isRunning()) - .ifPresent(c -> runImportantContainerMaintainers(context, c)); - - switch (node.state()) { - case ready, reserved, failed, inactive, parked -> { - storageMaintainer.syncLogs(context, true); - if (node.state() == NodeState.reserved) downloadImageIfNeeded(context, container); - removeContainerIfNeededUpdateContainerState(context, container); - updateNodeRepoWithCurrentAttributes(context, Optional.empty()); - stopServicesIfNeeded(context); - } - case active -> { - storageMaintainer.syncLogs(context, true); - storageMaintainer.cleanDiskIfFull(context); - storageMaintainer.handleCoreDumpsForContainer(context, container, false); - - if (downloadImageIfNeeded(context, container)) { - context.log(logger, "Waiting for image to download " + context.node().wantedDockerImage().get().asString()); - return; - } - dropDocsIfNeeded(context, container); - container = removeContainerIfNeededUpdateContainerState(context, container); - credentialsMaintainers.forEach(maintainer -> maintainer.converge(context)); - if (container.isEmpty()) { - containerState = STARTING; - container = Optional.of(startContainer(context)); - containerState = UNKNOWN; - runImportantContainerMaintainers(context, container.get()); - } else { - container = Optional.of(updateContainerIfNeeded(context, container.get())); - } - - serviceDumper.processServiceDumpRequest(context); - - startServicesIfNeeded(context); - resumeNodeIfNeeded(context); - if (healthChecker.isPresent()) { - healthChecker.get().verifyHealth(context); - if (firstSuccessfulHealthCheckInstant.isEmpty()) - firstSuccessfulHealthCheckInstant = Optional.of(timer.currentTime()); - - Duration timeLeft = Duration.between(timer.currentTime(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration(context))); - if ( ! container.get().resources().equalsCpu(getContainerResources(context)) - && context.node().currentDockerImage().isPresent()) // Immediately resume first-time deployments, when healthy. - throw ConvergenceException.ofTransient("Refusing to resume until warm up period ends (" + - (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")"); - } - - // Because it's more important to stop a bad release from rolling out in prod, - // we put the resume call last. So if we fail after updating the node repo attributes - // but before resume, the app may go through the tenant pipeline but will halt in prod. - // - // Note that this problem exists only because there are 2 different mechanisms - // that should really be parts of a single mechanism: - // - The content of node repo is used to determine whether a new Vespa+application - // has been successfully rolled out. - // - Slobrok and internal orchestrator state is used to determine whether - // to allow upgrade (suspend). - updateNodeRepoWithCurrentAttributes(context, container.map(Container::createdAt)); - if (suspendedInOrchestrator || node.orchestratorStatus().isSuspended()) { - context.log(logger, "Call resume against Orchestrator"); - orchestrator.resume(context.hostname().value()); - suspendedInOrchestrator = false; - } - } - case dirty -> { - removeContainerIfNeededUpdateContainerState(context, container); - context.log(logger, "State is " + node.state() + ", will delete application storage and mark node as ready"); - credentialsMaintainers.forEach(maintainer -> maintainer.clearCredentials(context)); - storageMaintainer.syncLogs(context, false); - storageMaintainer.archiveNodeStorage(context); - updateNodeRepoWithCurrentAttributes(context, Optional.empty()); - nodeRepository.setNodeState(context.hostname().value(), NodeState.ready); - } - default -> throw ConvergenceException.ofError("Unexpected state " + node.state().name()); - } - } - - private void runImportantContainerMaintainers(NodeAgentContext context, Container runningContainer) { - aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); - wireguardTasks.forEach(task -> task.converge(context, runningContainer.id())); - } - - private static void logChangesToNodeSpec(NodeAgentContext context, NodeSpec lastNode, NodeSpec node) { - StringBuilder builder = new StringBuilder(); - appendIfDifferent(builder, "state", lastNode, node, NodeSpec::state); - if (builder.length() > 0) { - context.log(logger, Level.INFO, "Changes to node: " + builder); - } - } - - private static <T> String fieldDescription(T value) { - return value == null ? "[absent]" : value.toString(); - } - - private static <T> void appendIfDifferent(StringBuilder builder, String name, NodeSpec oldNode, NodeSpec newNode, Function<NodeSpec, T> getter) { - T oldValue = oldNode == null ? null : getter.apply(oldNode); - T newValue = getter.apply(newNode); - if (!Objects.equals(oldValue, newValue)) { - if (builder.length() > 0) { - builder.append(", "); - } - builder.append(name).append(" ").append(fieldDescription(oldValue)).append(" -> ").append(fieldDescription(newValue)); - } - } - - private Optional<Container> getContainer(NodeAgentContext context) { - if (containerState == ABSENT) return Optional.empty(); - Optional<Container> container = containerOperations.getContainer(context); - if (container.isEmpty()) containerState = ABSENT; - return container; - } - - @Override - public int getAndResetNumberOfUnhandledExceptions() { - int temp = numberOfUnhandledException; - numberOfUnhandledException = 0; - return temp; - } - - private void orchestratorSuspendNode(NodeAgentContext context) { - if (context.node().state() != NodeState.active) return; - - context.log(logger, "Ask Orchestrator for permission to suspend node"); - orchestrator.suspend(context.hostname().value()); - suspendedInOrchestrator = true; - } - - protected void writeContainerData(NodeAgentContext context, ContainerData containerData) { } - - protected List<CredentialsMaintainer> credentialsMaintainers() { - return credentialsMaintainers; - } - - private Duration warmUpDuration(NodeAgentContext context) { - ZoneApi zone = context.zone(); - Optional<NodeMembership> membership = context.node().membership(); - return zone.getEnvironment().isTest() - || context.nodeType() != NodeType.tenant - || membership.map(mem -> ! (mem.type().hasContainer() || mem.type().isAdmin())).orElse(false) - ? Duration.ofSeconds(-1) - : warmUpDuration.dividedBy(zone.getSystemName().isCd() ? 3 : 1); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java deleted file mode 100644 index 59b3086988e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import java.time.Duration; -import java.time.Instant; - -/** - * @author freva - */ -public interface NodeAgentScheduler { - - /** Schedule a tick for NodeAgent to run with the given NodeAgentContext, at no earlier than given instant */ - void scheduleTickWith(NodeAgentContext context, Instant at); - - /** - * Will eventually freeze/unfreeze the node agent - * @param frozen whether node agent should be frozen - * @param timeout maximum duration this method should block while waiting for NodeAgent to reach target state - * @return True if node agent has converged to the desired state - */ - boolean setFrozen(boolean frozen, Duration timeout); - - /** @return the last scheduled context or a default value */ - NodeAgentContext currentContext(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java deleted file mode 100644 index 3e7895c1ebd..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -public enum NodeAgentTask { - - // The full task name is prefixed with 'node>', e.g. 'node>DiskCleanup' - DiskCleanup, - CoreDumps, - CredentialsMaintainer, - AclMaintainer; - - private static final Map<String, NodeAgentTask> tasksByName = Arrays.stream(NodeAgentTask.values()) - .collect(Collectors.toUnmodifiableMap(NodeAgentTask::taskName, n -> n)); - - private final String taskName; - NodeAgentTask() { - this.taskName = "node>" + name(); - } - - public String taskName() { return taskName; } - - public static Set<NodeAgentTask> fromString(List<String> tasks) { - return tasks.stream().filter(tasksByName::containsKey).map(tasksByName::get).collect(Collectors.toUnmodifiableSet()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/PathScope.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/PathScope.java deleted file mode 100644 index a8effa19b27..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/PathScope.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystem; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; - -import java.nio.file.Path; -import java.util.Objects; - -/** - * @author freva - */ -public class PathScope { - - private final ContainerFileSystem containerFs; - private final String pathToVespaHome; - private final UserScope users; - - public PathScope(ContainerFileSystem containerFs, String pathToVespaHome) { - this.containerFs = Objects.requireNonNull(containerFs); - this.pathToVespaHome = Objects.requireNonNull(pathToVespaHome); - this.users = containerFs.getUserPrincipalLookupService().userScope(); - } - - public ContainerPath of(String pathInNode) { - return of(pathInNode, users.root()); - } - - public ContainerPath of(String pathInNode, UnixUser user) { - return ContainerPath.fromPathInContainer(containerFs, Path.of(pathInNode), user); - } - - public ContainerPath underVespaHome(String relativePath) { - if (relativePath.startsWith("/")) - throw new IllegalArgumentException("Expected a relative path to the Vespa home, got: " + relativePath); - - return ContainerPath.fromPathInContainer(containerFs, Path.of(pathToVespaHome, relativePath), users.vespa()); - } - - public ContainerPath fromPathOnHost(Path pathOnHost) { - return ContainerPath.fromPathOnHost(containerFs, pathOnHost, users.root()); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PathScope pathScope = (PathScope) o; - return containerFs.equals(pathScope.containerFs) && pathToVespaHome.equals(pathScope.pathToVespaHome) && users.equals(pathScope.users); - } - - @Override - public int hashCode() { - return Objects.hash(containerFs, pathToVespaHome, users); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java deleted file mode 100644 index f44a19de36e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import java.util.Objects; - -/** - * @author freva - */ -public class UserNamespace { - - /** - * IDs outside the ID range are translated to the overflow ID before being written to disk: - * https://github.com/torvalds/linux/blob/5bfc75d92efd494db37f5c4c173d3639d4772966/Documentation/admin-guide/sysctl/fs.rst#overflowgid--overflowuid - * Real value in /proc/sys/fs/overflowuid or overflowgid, hardcode default value*/ - private static final int OVERFLOW_ID = 65_534; - - private final int uidOffset; - private final int gidOffset; - private final int idRangeSize; - - public UserNamespace(int uidOffset, int gidOffset, int idRangeSize) { - this.uidOffset = uidOffset; - this.gidOffset = gidOffset; - this.idRangeSize = idRangeSize; - } - - public int userIdOnHost(int containerUid) { return toHostId(containerUid, uidOffset, idRangeSize); } - public int groupIdOnHost(int containerGid) { return toHostId(containerGid, gidOffset, idRangeSize); } - public int userIdInContainer(int hostUid) { return toContainerId(hostUid, uidOffset, idRangeSize); } - public int groupIdInContainer(int hostGid) { return toContainerId(hostGid, gidOffset, idRangeSize); } - - public int idRangeSize() { return idRangeSize; } - public int overflowId() { return OVERFLOW_ID; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - UserNamespace that = (UserNamespace) o; - return uidOffset == that.uidOffset && gidOffset == that.gidOffset && idRangeSize == that.idRangeSize; - } - - @Override - public int hashCode() { - return Objects.hash(uidOffset, gidOffset, idRangeSize); - } - - @Override - public String toString() { - return "UserNamespace{" + - "uidOffset=" + uidOffset + - ", gidOffset=" + gidOffset + - ", idRangeSize=" + idRangeSize + - '}'; - } - - private static int toHostId(int containerId, int idOffset, int idRangeSize) { - if (containerId < 0 || containerId > idRangeSize) - throw new IllegalArgumentException("Invalid container id: " + containerId); - return idOffset + containerId; - } - - private static int toContainerId(int hostId, int idOffset, int idRangeSize) { - hostId = hostId - idOffset; - return hostId < 0 || hostId >= idRangeSize ? OVERFLOW_ID : hostId; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserScope.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserScope.java deleted file mode 100644 index 508adde5902..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserScope.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; - -import java.util.Objects; - -/** - * @author freva - */ -public class UserScope { - - private final UnixUser root; - private final UnixUser vespa; - private final UserNamespace namespace; - - private UserScope(UnixUser root, UnixUser vespa, UserNamespace namespace) { - this.root = Objects.requireNonNull(root); - this.vespa = Objects.requireNonNull(vespa); - this.namespace = Objects.requireNonNull(namespace); - } - - public UnixUser root() { - return root; - } - - public UnixUser vespa() { - return vespa; - } - - public UserNamespace namespace() { - return namespace; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - UserScope userScope = (UserScope) o; - return root.equals(userScope.root) && vespa.equals(userScope.vespa) && namespace.equals(userScope.namespace); - } - - @Override - public int hashCode() { - return Objects.hash(root, vespa, namespace); - } - - /** Creates user scope with default root and vespa user */ - public static UserScope create(UserNamespace namespace) { - return new UserScope(UnixUser.ROOT, UnixUser.VESPA, namespace); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java deleted file mode 100644 index 42310c7233f..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java deleted file mode 100644 index 59040abc4bf..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.provider; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -/** - * Class to make it easier to implement a NodeAdminDebugHandler: - * - Forward to sub-NodeAdminDebugHandlers with addHandler, - * - Specify constants with addConstant - * - Forwarding to methods that dynamically build debug objects with addThreadSafeSupplier. - * - * @author hakonhall - */ -public class DebugHandlerHelper implements NodeAdminDebugHandler { - private final ConcurrentMap<String, Supplier<Object>> suppliers = new ConcurrentHashMap<>(); - - public void addThreadSafeSupplier(String name, Supplier<Object> threadSafeSupplier) { - Supplier<Object> previousSupplier = suppliers.putIfAbsent(name, threadSafeSupplier); - if (previousSupplier != null) { - throw new IllegalArgumentException(name + " is already registered"); - } - } - - public void addHandler(String name, NodeAdminDebugHandler handler) { - addThreadSafeSupplier(name, handler::getDebugPage); - } - - public void addConstant(String name, String value) { - addThreadSafeSupplier(name, () -> value); - } - - public void remove(String name) { - Supplier<Object> supplier = suppliers.remove(name); - if (supplier == null) { - throw new IllegalArgumentException(name + " is not registered"); - } - } - - @Override - public Map<String, Object> getDebugPage() { - return suppliers.entrySet().stream().collect(Collectors.toMap( - Map.Entry::getKey, - entry -> entry.getValue().get())); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java deleted file mode 100644 index 2c38422e127..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.provider; - -import java.util.Map; - -/** - * Interface for supporting debug info to introspect e.g. internal state. - * - * @author hakonhall - */ -public interface NodeAdminDebugHandler { - /** - * The Object in the map values must be serializable with Jackson's ObjectMapper. - * May be called concurrently by different threads. - */ - Map<String, Object> getDebugPage(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java deleted file mode 100644 index 8c8dd618869..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.provider; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java deleted file mode 100644 index baf0142df4d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.logging.Logger; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists; -import static com.yahoo.yolean.Exceptions.uncheck; -import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; -import static java.util.stream.Collectors.joining; - -/** - * Rewrites default-env.txt files. - * - * @author bjorncs - */ -public class DefaultEnvWriter { - - private static final Logger logger = Logger.getLogger(DefaultEnvWriter.class.getName()); - - private final Map<String, Operation> operations = new LinkedHashMap<>(); - - public DefaultEnvWriter addOverride(String name, String value) { - return addOperation("override", name, value); - } - - public DefaultEnvWriter addFallback(String name, String value) { - return addOperation("fallback", name, value); - } - - public DefaultEnvWriter addUnset(String name) { - return addOperation("unset", name, null); - } - - private DefaultEnvWriter addOperation(String action, String name, String value) { - if (operations.containsKey(name)) { - throw new IllegalArgumentException(String.format("Operation on variable '%s' already added", name)); - } - operations.put(name, new Operation(action, name, value)); - return this; - } - - /** - * Updates or created a default-env.txt file - * - * @return true if the file was modified - */ - public boolean updateFile(TaskContext context, Path defaultEnvFile) { - List<String> currentDefaultEnvLines = ifExists(() -> Files.readAllLines(defaultEnvFile)).orElse(List.of()); - List<String> newDefaultEnvLines = generateContent(currentDefaultEnvLines); - if (currentDefaultEnvLines.equals(newDefaultEnvLines)) { - return false; - } else { - context.log(logger, "Updating " + defaultEnvFile.toString()); - Path tempFile = defaultEnvFile.resolveSibling(defaultEnvFile.getFileName() + ".tmp"); - uncheck(() -> Files.write(tempFile, newDefaultEnvLines)); - uncheck(() -> Files.move(tempFile, defaultEnvFile, ATOMIC_MOVE)); - return true; - } - } - - /** - * @return generated default-env.txt content - */ - public String generateContent() { - return generateContent(List.of()).stream() - .collect(joining(System.lineSeparator(), "", System.lineSeparator())); - } - - private List<String> generateContent(List<String> currentDefaultEnvLines) { - List<String> newDefaultEnvLines = new ArrayList<>(); - Set<String> seenNames = new TreeSet<>(); - for (String line : currentDefaultEnvLines) { - String[] items = line.split(" "); - if (items.length < 2) { - throw new IllegalArgumentException(String.format("Invalid line in file '%s': %s", currentDefaultEnvLines, line)); - } - String name = items[1]; - if (!seenNames.contains(name)) { // implicitly removes duplicated variables - seenNames.add(name); - Operation operation = operations.get(name); - if (operation != null) { - newDefaultEnvLines.add(operation.toLine()); - } else { - newDefaultEnvLines.add(line); - } - } - } - for (var operation : operations.values()) { - if (!seenNames.contains(operation.name)) { - newDefaultEnvLines.add(operation.toLine()); - } - } - return newDefaultEnvLines; - } - - private record Operation(String action, String name, String value) { - String toLine() { - if (action.equals("unset")) { - return "unset " + name; - } - return action + " " + name + " " + value; - } - } -} - - diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Cursor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Cursor.java deleted file mode 100644 index 4e9998bd40f..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Cursor.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.Optional; -import java.util.function.Function; -import java.util.regex.Pattern; - -/** - * Simulates an editor cursor. - * - * @author hakon - */ -public interface Cursor { - // CURSOR AND BUFFER QUERIES - - String getBufferText(); - String getLine(); - String getPrefix(); - String getSuffix(); - String getTextTo(Mark mark); - - Position getPosition(); - Mark createMark(); - - // CURSOR MOVEMENT - - Cursor moveToStartOfBuffer(); - Cursor moveToEndOfBuffer(); - - Cursor moveToStartOfLine(); - Cursor moveToStartOfPreviousLine(); - Cursor moveToStartOfNextLine(); - Cursor moveToStartOf(int lineIndex); - - Cursor moveToEndOfLine(); - Cursor moveToEndOfPreviousLine(); - Cursor moveToEndOfNextLine(); - Cursor moveToEndOf(int lineIndex); - - Cursor moveForward(); - Cursor moveForward(int times); - Cursor moveBackward(); - Cursor moveBackward(int times); - - Cursor moveTo(Mark mark); - Cursor moveTo(Position position); - Cursor moveTo(int lineIndex, int columnIndex); - - Optional<Match> moveForwardToStartOfMatch(Pattern pattern); - Optional<Match> moveForwardToEndOfMatch(Pattern pattern); - - boolean skipBackward(String text); - boolean skipForward(String text); - - // BUFFER MODIFICATIONS - - Cursor write(String text); - Cursor writeLine(String line); - Cursor writeLines(String... lines); - Cursor writeLines(Iterable<String> lines); - - Cursor writeNewline(); - Cursor writeNewlineAfter(); - - Cursor deleteAll(); - Cursor deleteLine(); - Cursor deletePrefix(); - Cursor deleteSuffix(); - - Cursor deleteForward(); - Cursor deleteForward(int times); - Cursor deleteBackward(); - Cursor deleteBackward(int times); - - Cursor deleteTo(Mark mark); - - boolean replaceMatch(Pattern pattern, Function<Match, String> replacer); - - /** - * Replace matches of a pattern. - * - * <p>The search for {@code pattern} starts at cursor and matches against the remaining line, - * and the full line for the following lines. Each match is replaced by a String returned by - * {@code replacer::apply}. - * - * <p>The cursor is unchanged without any matches, or moved to the end of the last replacement. - * - * <p>To replace all matches in a buffer, first call {@link #moveToStartOfBuffer()} to - * postion the cursor at the beginning of the buffer. - * - * @see #moveForwardToStartOfMatch(Pattern) - * @see #moveForwardToEndOfMatch(Pattern) - */ - int replaceMatches(Pattern pattern, Function<Match, String> replacer); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java deleted file mode 100644 index 501db764d05..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.List; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.regex.Pattern; - -import static com.yahoo.collections.Comparables.max; -import static com.yahoo.collections.Comparables.min; - -/** - * @author hakon - */ -public class CursorImpl implements Cursor { - private final TextBuffer textBuffer; - private final Object unique = new Object(); - - private Position position; - - /** - * Creates a cursor to a text buffer. - * - * WARNING: The text buffer MUST NOT be accessed outside this cursor. This cursor - * takes sole ownership of the text buffer. - * - * @param textBuffer the text buffer this cursor owns and operates on - */ - CursorImpl(TextBuffer textBuffer) { - this.textBuffer = textBuffer; - position = textBuffer.getStartOfText(); - } - - @Override - public Position getPosition() { - return position; - } - - @Override - public Mark createMark() { - return new Mark(position, textBuffer.getVersion(), unique); - } - - @Override - public String getBufferText() { - return textBuffer.getString(); - } - - @Override - public String getLine() { - return textBuffer.getLine(position); - } - - @Override - public String getPrefix() { - return textBuffer.getLinePrefix(position); - } - - @Override - public String getSuffix() { - return textBuffer.getLineSuffix(position); - } - - @Override - public String getTextTo(Mark mark) { - validateMark(mark); - - Position start = min(mark.position(), position); - Position end = max(mark.position(), position); - - return textBuffer.getSubstring(start, end); - } - - @Override - public Cursor moveToStartOfBuffer() { - position = textBuffer.getStartOfText(); - return this; - } - - @Override - public Cursor moveToEndOfBuffer() { - position = textBuffer.getEndOfText(); - return this; - } - - @Override - public Cursor moveToStartOfLine() { - position = textBuffer.getStartOfLine(position); - return this; - } - - @Override - public Cursor moveToStartOfPreviousLine() { - position = textBuffer.getStartOfPreviousLine(position); - return this; - } - - @Override - public Cursor moveToStartOfNextLine() { - position = textBuffer.getStartOfNextLine(position); - return this; - } - - @Override - public Cursor moveToStartOf(int lineIndex) { - validateLineIndex(lineIndex); - position = new Position(lineIndex, 0); - return this; - } - - @Override - public Cursor moveToEndOfLine() { - position = textBuffer.getEndOfLine(position); - return this; - } - - @Override - public Cursor moveToEndOfPreviousLine() { - return moveToStartOfPreviousLine().moveToEndOfLine(); - } - - @Override - public Cursor moveToEndOfNextLine() { - return moveToStartOfNextLine().moveToEndOfLine(); - } - - @Override - public Cursor moveToEndOf(int lineIndex) { - return moveToStartOf(lineIndex).moveToEndOfLine(); - } - - @Override - public Cursor moveForward() { - return moveForward(1); - } - - @Override - public Cursor moveForward(int times) { - position = textBuffer.forward(position, times); - return this; - } - - @Override - public Cursor moveBackward() { - return moveBackward(1); - } - - @Override - public Cursor moveBackward(int times) { - position = textBuffer.backward(position, times); - return this; - } - - @Override - public Cursor moveTo(Mark mark) { - validateMark(mark); - position = mark.position(); - return this; - } - - @Override - public boolean skipBackward(String text) { - String prefix = getPrefix(); - if (prefix.endsWith(text)) { - position = new Position(position.lineIndex(), position.columnIndex() - text.length()); - return true; - } else { - return false; - } - } - - @Override - public boolean skipForward(String text) { - String suffix = getSuffix(); - if (suffix.startsWith(text)) { - position = new Position(position.lineIndex(), position.columnIndex() + text.length()); - return true; - } else { - return false; - } - } - - @Override - public Optional<Match> moveForwardToStartOfMatch(Pattern pattern) { - return moveForwardToXOfMatch(pattern, match -> position = match.startOfMatch()); - } - - @Override - public Optional<Match> moveForwardToEndOfMatch(Pattern pattern) { - return moveForwardToXOfMatch(pattern, match -> position = match.endOfMatch()); - } - - private Optional<Match> moveForwardToXOfMatch(Pattern pattern, Consumer<Match> callback) { - Optional<Match> match = textBuffer.findForward(position, pattern); - match.ifPresent(callback); - return match; - } - - @Override - public Cursor moveTo(Position position) { - validatePosition(position); - this.position = position; - return this; - } - - @Override - public Cursor moveTo(int lineIndex, int columnIndex) { - return moveTo(new Position(lineIndex, columnIndex)); - } - - @Override - public Cursor write(String text) { - position = textBuffer.write(position, text); - return this; - } - - @Override - public Cursor writeLine(String line) { - return write(line).write("\n"); - } - - @Override - public Cursor writeLines(String... lines) { - return writeLines(List.of(lines)); - } - - @Override - public Cursor writeLines(Iterable<String> lines) { - return writeLine(String.join("\n", lines)); - } - - @Override - public Cursor writeNewline() { - return write("\n"); - } - - @Override - public Cursor writeNewlineAfter() { - return writeNewline().moveBackward(); - } - - @Override - public Cursor deleteAll() { - moveToStartOfBuffer(); - textBuffer.clear(); - return this; - } - - @Override - public Cursor deleteLine() { - moveToStartOfLine(); - textBuffer.delete(position, textBuffer.getStartOfNextLine(position)); - return this; - } - - @Override - public Cursor deletePrefix() { - Position originalPosition = position; - moveToStartOfLine(); - textBuffer.delete(position, originalPosition); - return this; - } - - @Override - public Cursor deleteSuffix() { - textBuffer.delete(position, textBuffer.getEndOfLine(position)); - return this; - } - - @Override - public Cursor deleteForward() { - return deleteForward(1); - } - - @Override - public Cursor deleteForward(int times) { - Position end = textBuffer.forward(position, times); - textBuffer.delete(position, end); - return this; - } - - @Override - public Cursor deleteBackward() { - return deleteBackward(1); - } - - @Override - public Cursor deleteBackward(int times) { - Position end = position; - moveBackward(times); - textBuffer.delete(position, end); - return this; - } - - @Override - public Cursor deleteTo(Mark mark) { - validateMark(mark); - Position start = min(mark.position(), position); - Position end = max(mark.position(), position); - - textBuffer.delete(start, end); - return this; - } - - @Override - public boolean replaceMatch(Pattern pattern, Function<Match, String> replacer) { - Optional<Match> match = moveForwardToStartOfMatch(pattern); - if (match.isEmpty()) { - return false; - } - - textBuffer.delete(match.get().startOfMatch(), match.get().endOfMatch()); - write(replacer.apply(match.get())); - return true; - } - - @Override - public int replaceMatches(Pattern pattern, Function<Match, String> replacer) { - int count = 0; - - for (; replaceMatch(pattern, replacer); ++count) { - // empty - } - - return count; - } - - private void validatePosition(Position position) { - validateLineIndex(position.lineIndex()); - - int maxColumnIndex = textBuffer.getLine(position.lineIndex()).length(); - if (position.columnIndex() < 0 || position.columnIndex() > maxColumnIndex) { - throw new IndexOutOfBoundsException("Column index of " + position.coordinateString() + - " is not in permitted range [0," + maxColumnIndex + "]"); - } - } - - private void validateLineIndex(int lineIndex) { - int maxLineIndex = textBuffer.getMaxLineIndex(); - if (lineIndex < 0 || lineIndex > maxLineIndex) { - throw new IndexOutOfBoundsException("Line index " + lineIndex + - " not in permitted range [0," + maxLineIndex + "]"); - } - } - - private void validateMark(Mark mark) { - if (mark.secret() != unique) { - throw new IllegalArgumentException("Unknown mark " + mark); - } - - if (!mark.version().equals(textBuffer.getVersion())) { - throw new IllegalArgumentException("Mark " + mark + " is outdated"); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/FileEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/FileEditor.java deleted file mode 100644 index fb09482a43a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/FileEditor.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.nio.file.Path; - -/** - * @author hakon - */ -public class FileEditor { - private final UnixPath path; - private final StringEditor stringEditor; - - private String fileText; - private Version fileVersion; - - public static FileEditor open(Path path) { - UnixPath unixPath = new UnixPath(path); - String text = unixPath.readUtf8File(); - StringEditor stringEditor = new StringEditor(text); - return new FileEditor(unixPath, text, stringEditor); - } - - private FileEditor(UnixPath path, String fileText, StringEditor stringEditor) { - this.path = path; - this.fileText = fileText; - this.stringEditor = stringEditor; - fileVersion = stringEditor.bufferVersion(); - } - - public Cursor cursor() { - return stringEditor.cursor(); - } - - public void reloadFile() { - fileText = path.readUtf8File(); - stringEditor.cursor().deleteAll().write(fileText); - fileVersion = stringEditor.bufferVersion(); - } - - public boolean save() { - Version bufferVersion = stringEditor.bufferVersion(); - if (bufferVersion.equals(fileVersion)) { - return false; - } - - String newText = stringEditor.cursor().getBufferText(); - if (newText.equals(fileText)) { - return false; - } - - path.writeUtf8File(newText); - fileVersion = bufferVersion; - return true; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java deleted file mode 100644 index 616c98c5b76..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.Objects; - -/** - * @author hakon - */ -public class Mark { - private final Position position; - private final Version version; - private final Object token; - - Mark(Position position, Version version, Object token) { - this.position = position; - this.version = version; - this.token = token; - } - - public Position position() { - return position; - } - - public Version version() { - return version; - } - - public Object secret() { - return token; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Mark mark = (Mark) o; - return Objects.equals(position, mark.position) && - Objects.equals(version, mark.version) && - token == mark.token; - } - - @Override - public int hashCode() { - return Objects.hash(position, version, token); - } - - @Override - public String toString() { - return position.coordinateString() + "@" + version; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Match.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Match.java deleted file mode 100644 index 32e058c1067..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Match.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.regex.Matcher; - -/** - * Represents a pattern match of a line - * - * @author hakon - */ -public class Match { - private final int lineIndex; - private final String line; - private final Matcher matcher; - - Match(int lineIndex, String line, Matcher matcher) { - this.lineIndex = lineIndex; - this.line = line; - this.matcher = matcher; - } - - /** The part of the line before the match */ - public String prefix() { - return line.substring(0, matcher.start()); - } - - /** The part of the line that matched */ - public String match() { - return matcher.group(); - } - - /** The part of the line that followed the match */ - public String suffix() { - return line.substring(matcher.end()); - } - - public Position startOfMatch() { - return new Position(lineIndex, matcher.start()); - } - - public Position endOfMatch() { - return new Position(lineIndex, matcher.end()); - } - - public int groupCount() { - return matcher.groupCount(); - } - - public String group(int groupnr) { - return matcher.group(groupnr); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java deleted file mode 100644 index 95aa778d57e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.Comparator; -import java.util.Objects; - -/** - * Represents a position in the buffer - * - * @author hakon - */ -public class Position implements Comparable<Position> { - private static final Position START_POSITION = new Position(0, 0); - - private static final Comparator<Position> COMPARATOR = Comparator - .comparingInt(Position::lineIndex) - .thenComparingInt(Position::columnIndex); - - private final int lineIndex; - private final int columnIndex; - - /** Returns the first position at line index 0 and column index 0 */ - public static Position start() { - return START_POSITION; - } - - Position(int lineIndex, int columnIndex) { - this.lineIndex = lineIndex; - this.columnIndex = columnIndex; - } - - public int lineIndex() { - return lineIndex; - } - - public int columnIndex() { - return columnIndex; - } - - @Override - public int compareTo(Position that) { - return COMPARATOR.compare(this, that); - } - - public boolean isAfter(Position that) { return compareTo(that) > 0; } - public boolean isNotAfter(Position that) { return !isAfter(that); } - public boolean isBefore(Position that) { return compareTo(that) < 0; } - public boolean isNotBefore(Position that) { return !isBefore(that); } - - public String coordinateString() { - return "(" + lineIndex + "," + columnIndex + ")"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Position position = (Position) o; - return lineIndex == position.lineIndex && - columnIndex == position.columnIndex; - } - - @Override - public int hashCode() { - return Objects.hash(lineIndex, columnIndex); - } - - @Override - public String toString() { - return coordinateString(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditor.java deleted file mode 100644 index ea55e3c11a4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditor.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -/** - * Edits multi-line text. - * - * @author hakon - */ -public class StringEditor { - private final TextBuffer textBuffer; - private final Cursor cursor; - - public StringEditor() { - this(""); - } - - public StringEditor(String text) { - textBuffer = new TextBufferImpl(text); - cursor = new CursorImpl(textBuffer); - } - - public Cursor cursor() { - return cursor; - } - - public Version bufferVersion() { - return textBuffer.getVersion(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBuffer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBuffer.java deleted file mode 100644 index e6cf211d481..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBuffer.java +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author hakon - */ -interface TextBuffer { - // INTERFACE TO IMPLEMENT BY CONCRETE CLASS - - /** Get the version of the buffer - edits increment the version. */ - Version getVersion(); - - /** Return the text as a single String (likely) with embedded newlines. */ - String getString(); - - /** Return the maximum line index (the minimum line index is 0). */ - int getMaxLineIndex(); - - /** @param lineIndex must be in in {@code [0, getMaxLineIndex()]} */ - String getLine(int lineIndex); - - /** Insert the possibly multi-line text at position and return the end position. */ - Position write(Position position, String text); - - /** Delete everything. */ - void clear(); - - /** Delete range. */ - void delete(Position start, Position end); - - // DERIVED IMPLEMENTATION - - /** - * Return the Position closest to {@code position} which is in the range - * {@code [getStartOfText(), getEndOfText()]}. - */ - default Position getValidPositionClosestTo(Position position) { - if (position.isBefore(getStartOfText())) { - return getStartOfText(); - } else if (position.isAfter(getEndOfText())) { - return getEndOfText(); - } else { - return position; - } - } - - default String getLine(Position position) { return getLine(position.lineIndex()); } - - default String getLinePrefix(Position position) { - return getLine(position.lineIndex()).substring(0, position.columnIndex()); - } - - default String getLineSuffix(Position position) { - return getLine(position.lineIndex()).substring(position.columnIndex()); - } - - default String getSubstring(Position start, Position end) { - if (start.lineIndex() < end.lineIndex()) { - StringBuilder builder = new StringBuilder(getLineSuffix(start)); - for (int i = start.lineIndex() + 1; i < end.lineIndex(); ++i) { - builder.append('\n'); - builder.append(getLine(i)); - } - return builder.append('\n').append(getLinePrefix(end)).toString(); - } else if (start.lineIndex() == end.lineIndex() && start.columnIndex() <= end.columnIndex()) { - return getLine(start).substring(start.columnIndex(), end.columnIndex()); - } - - throw new IllegalArgumentException( - "Bad range [" + start.coordinateString() + "," + end.coordinateString() + ">"); - } - - default Position getStartOfText() { return Position.start(); } // aka (0,0) - - default Position getEndOfText() { - int maxLineIndex = getMaxLineIndex(); - return new Position(maxLineIndex, getLine(maxLineIndex).length()); - } - - default Position getStartOfLine(Position position) { - return new Position(position.lineIndex(), 0); - } - - default Position getEndOfLine(Position position) { - return new Position(position.lineIndex(), getLine(position).length()); - } - - default Position getStartOfNextLine(Position position) { - if (position.lineIndex() < getMaxLineIndex()) { - return new Position(position.lineIndex() + 1, 0); - } else { - return getEndOfText(); - } - } - - default Position getStartOfPreviousLine(Position position) { - int lineIndex = position.lineIndex(); - if (lineIndex > 0) { - return new Position(lineIndex - 1, 0); - } else { - return getStartOfText(); - } - } - - default Position forward(Position position, int length) { - int lineIndex = position.lineIndex(); - int columnIndex = position.columnIndex(); - - int offsetLeft = length; - do { - String line = getLine(lineIndex); - int columnIndexWithInfiniteLine = columnIndex + offsetLeft; - if (columnIndexWithInfiniteLine <= line.length()) { - return new Position(lineIndex, columnIndexWithInfiniteLine); - } else if (lineIndex >= getMaxLineIndex()) { - // End of text - return new Position(lineIndex, line.length()); - } - - offsetLeft -= line.length() - columnIndex; - - // advance past newline - --offsetLeft; - ++lineIndex; - columnIndex = 0; - - // At this point: offsetLeft is guaranteed to be >= 0, and lineIndex <= max line index - } while (true); - } - - default Position backward(Position position, int length) { - int lineIndex = position.lineIndex(); - int columnIndex = position.columnIndex(); - - int offsetLeft = length; - do { - int columnIndexWithInfiniteLine = columnIndex - offsetLeft; - if (columnIndexWithInfiniteLine >= 0) { - return new Position(lineIndex, columnIndexWithInfiniteLine); - } else if (lineIndex <= 0) { - // Start of text - return new Position(0, 0); - } - - offsetLeft -= columnIndex; - - // advance past newline - --offsetLeft; - --lineIndex; - columnIndex = getLine(lineIndex).length(); - - // At this point: offsetLeft is guaranteed to be <= 0, and lineIndex >= 0 - } while (true); - } - - default Optional<Match> findForward(Position startPosition, Pattern pattern) { - for (Position position = startPosition; ; position = getStartOfNextLine(position)) { - String line = getLine(position); - Matcher matcher = pattern.matcher(line); - if (matcher.find(position.columnIndex())) { - return Optional.of(new Match(position.lineIndex(), line, matcher)); - } - - if (position.lineIndex() == getMaxLineIndex()) { - // search failed - no lines matched - return Optional.empty(); - } - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImpl.java deleted file mode 100644 index 0a7ff26c73c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImpl.java +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.task.util.editor.TextUtil.splitString; - -/** - * @author hakon - */ -public class TextBufferImpl implements TextBuffer { - /** Invariant: {@code size() >= 1}. An empty text buffer {@code => [""]} */ - private final ArrayList<String> lines = new ArrayList<>(); - - private Version version = new Version(); - - TextBufferImpl() { - lines.add(""); - } - - TextBufferImpl(String text) { - this(); - write(getStartOfText(), text); - // reset version - version = new Version(); - } - - @Override - public Version getVersion() { - return version; - } - - @Override - public String getString() { - return String.join("\n", lines); - } - - @Override - public int getMaxLineIndex() { - return lines.size() - 1; - } - - @Override - public String getLine(int lineIndex) { - return lines.get(lineIndex); - } - - @Override - public Position write(Position position, String text) { - List<String> linesToInsert = new LinkedList<>(splitString(text, true, false)); - if (linesToInsert.isEmpty()) { - return position; - } - - // The position splits that line in two, and both prefix and suffix must be preserved - linesToInsert.set(0, getLinePrefix(position) + linesToInsert.get(0)); - String lastLine = linesToInsert.get(linesToInsert.size() - 1); - int endColumnIndex = lastLine.length(); - linesToInsert.set(linesToInsert.size() - 1, lastLine + getLineSuffix(position)); - - // Set the first line at lineIndex, insert the rest. - int lineIndex = position.lineIndex(); - int endLineIndex = lineIndex + linesToInsert.size() - 1; - lines.set(lineIndex, linesToInsert.remove(0)); - lines.addAll(lineIndex + 1, linesToInsert); - - incrementVersion(); - - return new Position(endLineIndex, endColumnIndex); - } - - @Override - public void clear() { - lines.clear(); - lines.add(""); - } - - @Override - public void delete(Position start, Position end) { - if (start.isAfter(end)) { - throw new IllegalArgumentException("start position " + start + - " is after end position " + end); - } - - String prefix = getLinePrefix(start); - String suffix = getLineSuffix(end); - String stichedLine = prefix + suffix; - - lines.set(start.lineIndex(), stichedLine); - - deleteLines(start.lineIndex() + 1, end.lineIndex() + 1); - - incrementVersion(); - } - - private void deleteLines(int startIndex, int endIndex) { - for (int fromIndex = endIndex, toIndex = startIndex; fromIndex <= getMaxLineIndex(); - ++toIndex, ++fromIndex) { - lines.set(toIndex, lines.get(fromIndex)); - } - - truncate(getMaxLineIndex() - (endIndex - startIndex)); - } - - private void truncate(int newMaxLineIndex) { - while (getMaxLineIndex() > newMaxLineIndex) { - lines.remove(getMaxLineIndex()); - } - } - - private void incrementVersion() { - version = version.next(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextUtil.java deleted file mode 100644 index 625bb608fd7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Consumer; - -/** - * @author hakon - */ -public class TextUtil { - private TextUtil() {} - - /** - * Splits {@code text} by newline (LF {@code '\n'}). - * - * @param text the text to split into lines - * @param empty whether an empty text implies an empty List (true), or a List with one - * empty String element (false) - * @param prune whether a text ending with a newline will result in a List ending with the - * preceding line (true), or to add an empty String element (false) - */ - public static List<String> splitString(String text, boolean empty, boolean prune) { - List<String> lines = new ArrayList<>(); - splitString(text, empty, prune, lines::add); - return lines; - } - - /** - * Splits text by newline, passing each line to a consumer. - * - * @see #splitString(String, boolean, boolean) - */ - public static void splitString(String text, - boolean empty, - boolean prune, - Consumer<String> consumer) { - if (text.isEmpty()) { - if (!empty) { - consumer.accept(text); - } - return; - } - - final int endIndex = text.length(); - - int start = 0; - for (int end = text.indexOf('\n'); - end != -1; - start = end + 1, end = text.indexOf('\n', start)) { - consumer.accept(text.substring(start, end)); - } - - if (start < endIndex || !prune) { - consumer.accept(text.substring(start)); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java deleted file mode 100644 index 97d8cbb6a50..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import java.util.Objects; - -/** - * Represents a snapshot of the TextBuffer, between two edits (or the initial or final state) - * - * @author hakon - */ -public class Version { - private final int version; - - Version() { - this(0); - } - - private Version(int version) { - this.version = version; - } - - public boolean isBefore(Version that) { - return version < that.version; - } - - public int asInt() { - return version; - } - - public Version next() { - return new Version(version + 1); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Version that = (Version) o; - return version == that.version; - } - - @Override - public int hashCode() { - return Objects.hash(version); - } - - @Override - public String toString() { - return String.valueOf(version); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java deleted file mode 100644 index 73eddd2bbe2..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.file.Path; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.logging.Logger; - -/** - * Class to converge file/directory attributes like owner and permissions to wanted values. - * Typically used by higher abstraction layers working on files (FileSync/FileWriter) or - * directories (MakeDirectory). - * - * @author hakonhall - */ -public class AttributeSync { - private static final Logger logger = Logger.getLogger(AttributeSync.class.getName()); - - private final UnixPath path; - - private Optional<Integer> ownerId = Optional.empty(); - private Optional<Integer> groupId = Optional.empty(); - private Optional<String> permissions = Optional.empty(); - - public AttributeSync(Path path) { - this.path = new UnixPath(path); - } - - public Optional<String> getPermissions() { - return permissions; - } - - public AttributeSync withPermissions(String permissions) { - this.permissions = Optional.of(permissions); - return this; - } - - public Optional<Integer> ownerId() { - return ownerId; - } - - public AttributeSync withOwnerId(int ownerId) { - this.ownerId = Optional.of(ownerId); - return this; - } - - public Optional<Integer> groupId() { - return groupId; - } - - public AttributeSync withGroupId(int groupId) { - this.groupId = Optional.of(groupId); - return this; - } - - public AttributeSync with(PartialFileData fileData) { - ownerId = fileData.getOwnerId(); - groupId = fileData.getGroupId(); - permissions = fileData.getPermissions(); - return this; - } - - public boolean converge(TaskContext context) { - return converge(context, new FileAttributesCache(path)); - } - - /** - * Path must exist before calling converge. - */ - public boolean converge(TaskContext context, FileAttributesCache currentAttributes) { - boolean systemModified = updateAttribute( - context, - "user ID", - ownerId, - () -> currentAttributes.getOrThrow().ownerId(), - path::setOwnerId); - - systemModified |= updateAttribute( - context, - "group ID", - groupId, - () -> currentAttributes.getOrThrow().groupId(), - path::setGroupId); - - systemModified |= updateAttribute( - context, - "permissions", - permissions, - () -> currentAttributes.getOrThrow().permissions(), - path::setPermissions); - - return systemModified; - } - - private <T> boolean updateAttribute(TaskContext context, - String attributeName, - Optional<T> wantedValue, - Supplier<T> currentValueSupplier, - Consumer<T> valueSetter) { - if (wantedValue.isEmpty()) { - return false; - } - - T currentValue = currentValueSupplier.get(); - if (Objects.equals(currentValue, wantedValue.get())) { - return false; - } - - context.recordSystemModification( - logger, - String.format("Changing %s of %s from %s to %s", - attributeName, - path, - currentValue, - wantedValue.get())); - - valueSetter.accept(wantedValue.get()); - - return true; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java deleted file mode 100644 index b1fedd47e60..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.util.Locale; -import java.util.Objects; - -/** - * @author freva - */ -public class DiskSize implements Comparable<DiskSize> { - - public static final DiskSize ZERO = DiskSize.of(0); - private static final char[] UNITS = "kMGTPE".toCharArray(); - - public enum Unit { - kB(1_000), kiB(1 << 10), - MB(1_000_000), MiB(1 << 20), - GB(1_000_000_000), GiB(1 << 30), - TB(1_000_000_000_000L), TiB(1L << 40), - PB(1_000_000_000_000_000L), PiB(1L << 50); - - private final long size; - Unit(long size) { this.size = size; } - } - - private final long bytes; - private DiskSize(long bytes) { this.bytes = bytes; } - - public long bytes() { return bytes; } - public long as(Unit unit) { return bytes / unit.size; } - public double asDouble(Unit unit) { return (double) bytes / unit.size; } - - public DiskSize add(DiskSize other) { return new DiskSize(bytes + other.bytes); } - - public static DiskSize of(long bytes) { return new DiskSize(bytes); } - public static DiskSize of(double bytes, Unit unit) { return new DiskSize((long) (bytes * unit.size)); } - public static DiskSize of(long bytes, Unit unit) { return new DiskSize(bytes * unit.size); } - - public String asString() { return asString(0); } - public String asString(int decimals) { - if (bytes < 1000) return bytes + " bytes"; - - int unit = -1; - double remaining = bytes; - for (; remaining >= 1000; unit++) remaining /= 1000; - return String.format(Locale.ENGLISH, "%." + decimals + "f %sB", remaining, UNITS[unit]); - } - - @Override - public int compareTo(DiskSize rhs) { - return Long.compare(this.bytes, rhs.bytes); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DiskSize size = (DiskSize) o; - return bytes == size.bytes; - } - - @Override - public int hashCode() { - return Objects.hash(bytes); - } - - @Override - public String toString() { - return asString(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java deleted file mode 100644 index 66269602afd..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.LinkedList; -import java.util.List; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.logging.Logger; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists; -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * An editor meant to edit small line-based files like /etc/fstab. - * - * @author hakonhall - */ -public class Editor { - private static final Logger logger = Logger.getLogger(Editor.class.getName()); - private static final Charset ENCODING = StandardCharsets.UTF_8; - - private static final int MAX_LENGTH = 500; - - private final Supplier<List<String>> supplier; - private final Consumer<List<String>> consumer; - private final String name; - private final LineEditor editor; - private int diffSize = 0; - - /** - * Read the file which must be encoded in UTF-8, use the LineEditor to edit it, - * and any modifications were done write it back and return true. - */ - public Editor(Path path, LineEditor editor) { - this(path.toString(), - () -> ifExists(() -> Files.readAllLines(path, ENCODING)).orElseGet(List::of), - (newLines) -> uncheck(() -> Files.write(path, newLines, ENCODING)), - editor); - } - - /** - * @param name The name of what is being edited - used in logging - * @param supplier Supplies the editor with a list of lines to edit - * @param consumer Consumes the lines to presist if any changes is detected - * @param editor The line operations to execute on the lines supplied - */ - public Editor(String name, - Supplier<List<String>> supplier, - Consumer<List<String>> consumer, - LineEditor editor) { - this.supplier = supplier; - this.consumer = consumer; - this.name = name; - this.editor = editor; - } - - public boolean edit(Consumer<String> logConsumer) { - List<String> lines = supplier.get(); - List<String> newLines = new LinkedList<>(); - StringBuilder diff = new StringBuilder(); - boolean modified = false; - - for (String line : lines) { - LineEdit edit = editor.edit(line); - if (!edit.prependLines().isEmpty()) { - modified = true; - maybeAdd(diff, edit.prependLines()); - newLines.addAll(edit.prependLines()); - } - - switch (edit.getType()) { - case REPLACE -> { - modified = true; - maybeRemove(diff, line); - } - case NONE -> newLines.add(line); - default -> throw new IllegalArgumentException("Unknown EditType " + edit.getType()); - } - - if (!edit.appendLines().isEmpty()) { - modified = true; - maybeAdd(diff, edit.appendLines()); - newLines.addAll(edit.appendLines()); - } - } - - List<String> linesToAppend = editor.onComplete(); - if (!linesToAppend.isEmpty()) { - modified = true; - newLines.addAll(linesToAppend); - maybeAdd(diff, linesToAppend); - } - - if (!modified) { - return false; - } - - String diffDescription = diffTooLarge() ? ": Diff too large (" + diffSize + ")" : ":\n" + diff; - logConsumer.accept("Patching file " + name + diffDescription); - consumer.accept(newLines); - return true; - } - - public boolean converge(TaskContext context) { - return this.edit(line -> context.recordSystemModification(logger, line)); - } - - private void maybeAdd(StringBuilder diff, List<String> lines) { - for (String line : lines) { - // 2 for '+' and '\n' - diffSize += 2 + line.length(); - if (!diffTooLarge()) { - diff.append('+').append(line).append('\n'); - } - } - } - - private void maybeRemove(StringBuilder diff, String line) { - // 2 for '-' and '\n' - diffSize += 2 + line.length(); - if (!diffTooLarge()) { - diff.append('-').append(line).append('\n'); - } - } - - private boolean diffTooLarge() { - return diffSize > MAX_LENGTH; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorFactory.java deleted file mode 100644 index 66f54fc1967..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorFactory.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.nio.file.Path; - -/** - * @author hakonhall - */ -public class EditorFactory { - public Editor create(Path path, LineEditor editor) { - return new Editor(path, editor); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java deleted file mode 100644 index 06490bac3a4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.nio.file.attribute.FileTime; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.time.Instant; -import java.util.Map; -import java.util.Set; - -/** - * This wrapper around PosixFileAttributes. - * - * @author hakonhall - */ -public record FileAttributes(Instant lastModifiedTime, int ownerId, int groupId, String permissions, - boolean isRegularFile, boolean isDirectory, long size, int deviceMajor, int deviceMinor) { - - @SuppressWarnings("unchecked") - static FileAttributes fromAttributes(Map<String, Object> attributes) { - long dev_t = (long) attributes.get("dev"); - - return new FileAttributes( - ((FileTime) attributes.get("lastModifiedTime")).toInstant(), - (int) attributes.get("uid"), - (int) attributes.get("gid"), - PosixFilePermissions.toString(((Set<PosixFilePermission>) attributes.get("permissions"))), - (boolean) attributes.get("isRegularFile"), - (boolean) attributes.get("isDirectory"), - (long) attributes.get("size"), - deviceMajor(dev_t), deviceMinor(dev_t)); - } - - // Encoded as MMMM Mmmm mmmM MMmm, where M is a hex digit of the major number and m is a hex digit of the minor number. - static int deviceMajor(long dev_t) { return (int) (((dev_t & 0xFFFFF00000000000L) >> 32) | ((dev_t & 0xFFF00) >> 8)); } - static int deviceMinor(long dev_t) { return (int) (((dev_t & 0x00000FFFFFF00000L) >> 12) | (dev_t & 0x000FF)); } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java deleted file mode 100644 index ca81669adcc..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCache.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.util.Optional; - -// @ThreadUnsafe -public class FileAttributesCache { - private final UnixPath path; - - private Optional<FileAttributes> attributes = Optional.empty(); - - public FileAttributesCache(UnixPath path) { - this.path = path; - } - - public Optional<FileAttributes> get() { - if (attributes.isEmpty()) { - attributes = path.getAttributesIfExists(); - } - - return attributes; - } - - public FileAttributes getOrThrow() { - return get().orElseThrow(); - } - - public Optional<FileAttributes> forceGet() { - attributes = Optional.empty(); - return get(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java deleted file mode 100644 index 0a081ac53b4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCache.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.time.Instant; -import java.util.Optional; - -/** - * Class to avoid repeated reads of file content when the file seldom changes. - * - * @author hakonhall - */ -class FileContentCache { - private final UnixPath path; - - private Optional<byte[]> value = Optional.empty(); - private Optional<Instant> modifiedTime = Optional.empty(); - - FileContentCache(UnixPath path) { - this.path = path; - } - - byte[] get(Instant lastModifiedTime) { - if (modifiedTime.isEmpty() || lastModifiedTime.isAfter(modifiedTime.get())) { - value = Optional.of(path.readBytes()); - modifiedTime = Optional.of(lastModifiedTime); - } - - return value.get(); - } - - void updateWith(byte[] content, Instant modifiedTime) { - this.value = Optional.of(content); - this.modifiedTime = Optional.of(modifiedTime); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java deleted file mode 100644 index a443e683df0..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleter.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.logging.Logger; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Deletes a file or empty directory. - * - * @author hakonhall - */ -public class FileDeleter { - private static final Logger logger = Logger.getLogger(FileDeleter.class.getName()); - - private final Path path; - - public FileDeleter(Path path) { - this.path = path; - } - - public boolean converge(TaskContext context) { - boolean deleted = uncheck(() -> Files.deleteIfExists(path)); - if (deleted) { - context.recordSystemModification(logger, "Deleted " + path); - } - - return deleted; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java deleted file mode 100644 index 1b3fa1854e7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.lang.MutableInteger; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Deque; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; -import java.util.function.Predicate; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -/** - * Helper class to find and list or deleteRecursively files and directories. Follows the general syntax of command line - * tool `find`. - * - * @author freva - */ -public class FileFinder { - private static final Logger logger = Logger.getLogger(FileFinder.class.getName()); - - private final Path basePath; - private final Set<Path> pruned = new HashSet<>(); - private Predicate<FileAttributes> matcher; - private int maxDepth = Integer.MAX_VALUE; - - private FileFinder(Path basePath, Predicate<FileAttributes> initialMatcher) { - this.basePath = basePath; - this.matcher = initialMatcher; - } - - /** Creates a FileFinder at the given basePath */ - public static FileFinder from(Path basePath) { - return new FileFinder(basePath, attrs -> true); - } - - /** Creates a FileFinder at the given basePath that will match all files */ - public static FileFinder files(Path basePath) { - return new FileFinder(basePath, FileAttributes::isRegularFile); - } - - - /** Creates a FileFinder at the given basePath that will match all directories */ - public static FileFinder directories(Path basePath) { - return new FileFinder(basePath, FileAttributes::isDirectory); - } - - - /** - * Predicate that will be used to match files and directories under the base path. - * - * NOTE: Consecutive calls to this method are ANDed (this include the initial filter from - * {@link #files(Path)} or {@link #directories(Path)}. - */ - public FileFinder match(Predicate<FileAttributes> matcher) { - this.matcher = this.matcher.and(matcher); - return this; - } - - /** - * Path for which whole directory tree will be skipped, including the path itself. - * The path must be under {@code basePath} or be relative to {@code basePath}. - */ - public FileFinder prune(Path path) { - if (!path.isAbsolute()) - path = basePath.resolve(path); - - if (!path.startsWith(basePath)) - throw new IllegalArgumentException("Prune path " + path + " is not under base path " + basePath); - - this.pruned.add(path); - return this; - } - - /** Convenience method for pruning multiple paths, see {@link #prune(Path)}. */ - public FileFinder prune(Collection<Path> paths) { - paths.forEach(this::prune); - return this; - } - - /** - * Maximum depth (relative to basePath) where contents should be matched with the given filters. - * Default is unlimited. - */ - public FileFinder maxDepth(int maxDepth) { - this.maxDepth = maxDepth; - return this; - } - - /** - * Recursively deletes all matching elements - * - * @return true iff anything was matched and deleted - */ - public boolean deleteRecursively(TaskContext context) { - final int maxNumberOfDeletedPathsToLog = 20; - MutableInteger numDeleted = new MutableInteger(0); - List<Path> deletedPaths = new ArrayList<>(); - - try { - forEach(attributes -> { - if (attributes.unixPath().deleteRecursively()) { - if (numDeleted.next() <= maxNumberOfDeletedPathsToLog) deletedPaths.add(attributes.path()); - } - }); - } finally { - if (numDeleted.get() > maxNumberOfDeletedPathsToLog) { - context.log(logger, "Deleted " + numDeleted.get() + " paths under " + basePath); - } else if (deletedPaths.size() > 0) { - List<Path> paths = deletedPaths.stream() - .map(basePath::relativize) - .sorted() - .toList(); - context.log(logger, "Deleted these paths in " + basePath + ": " + paths); - } - } - - return deletedPaths.size() > 0; - } - - public List<FileAttributes> list() { - LinkedList<FileAttributes> list = new LinkedList<>(); - forEach(list::add); - return list; - } - - public Stream<FileAttributes> stream() { - return list().stream(); - } - - public void forEachPath(Consumer<Path> action) { - forEach(attributes -> action.accept(attributes.path())); - } - - /** Applies a given consumer to all the matching {@link FileFinder.FileAttributes} */ - public void forEach(Consumer<FileAttributes> action) { - applyForEachToMatching(basePath, matcher, maxDepth, action); - } - - - /** - * <p> This method walks a file tree rooted at a given starting file. The file tree traversal is - * <em>depth-first</em>: The filter function is applied in pre-order (NLR), but the given - * {@link Consumer} will be called in post-order (LRN). - */ - private void applyForEachToMatching(Path basePath, Predicate<FileAttributes> matcher, - int maxDepth, Consumer<FileAttributes> action) { - try { - // Only need to traverse as deep as we want to match, unless we want to match everything in directories - // already matched - Files.walkFileTree(basePath, Set.of(), maxDepth, new SimpleFileVisitor<>() { - private final Deque<FileAttributes> matchingDirectoryStack = new ArrayDeque<>(); - private int currentLevel = -1; - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - if (pruned.contains(dir)) return FileVisitResult.SKIP_SUBTREE; - - currentLevel++; - - FileAttributes attributes = new FileAttributes(dir, attrs); - if (currentLevel > 0 && matcher.test(attributes)) - matchingDirectoryStack.push(attributes); - - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - // When we find a directory at the max depth given to Files.walkFileTree, the directory - // will be passed to visitFile() rather than (pre|post)VisitDirectory - if (attrs.isDirectory()) { - preVisitDirectory(file, attrs); - return postVisitDirectory(file, null); - } - - FileAttributes attributes = new FileAttributes(file, attrs); - if (matcher.test(attributes)) - action.accept(attributes); - - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) { - if (!matchingDirectoryStack.isEmpty()) - action.accept(matchingDirectoryStack.pop()); - - currentLevel--; - return FileVisitResult.CONTINUE; - } - }); - } catch (NoSuchFileException ignored) { - - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - - // Ideally, we would reuse the FileAttributes in this package, but unfortunately we only get - // BasicFileAttributes and not PosixFileAttributes from FileVisitor - public static class FileAttributes { - private final Path path; - private final BasicFileAttributes attributes; - - public FileAttributes(Path path, BasicFileAttributes attributes) { - this.path = path; - this.attributes = attributes; - } - - public Path path() { return path; } - public UnixPath unixPath() { return new UnixPath(path); } - public String filename() { return path.getFileName().toString(); } - public Instant lastModifiedTime() { return attributes.lastModifiedTime().toInstant(); } - public boolean isRegularFile() { return attributes.isRegularFile(); } - public boolean isDirectory() { return attributes.isDirectory(); } - public long size() { return attributes.size(); } - } - - - // Filters - public static Predicate<FileAttributes> olderThan(Duration duration) { - return attrs -> Duration.between(attrs.lastModifiedTime(), Instant.now()).compareTo(duration) > 0; - } - - public static Predicate<FileAttributes> youngerThan(Duration duration) { - return olderThan(duration).negate(); - } - - public static Predicate<FileAttributes> largerThan(long sizeInBytes) { - return attrs -> attrs.size() > sizeInBytes; - } - - public static Predicate<FileAttributes> smallerThan(long sizeInBytes) { - return largerThan(sizeInBytes).negate(); - } - - public static Predicate<FileAttributes> nameMatches(Pattern pattern) { - return attrs -> pattern.matcher(attrs.filename()).matches(); - } - - public static Predicate<FileAttributes> nameStartsWith(String string) { - return attrs -> attrs.filename().startsWith(string); - } - - public static Predicate<FileAttributes> nameEndsWith(String string) { - return attrs -> attrs.filename().endsWith(string); - } - - public static Predicate<FileAttributes> all() { - return attrs -> true; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java deleted file mode 100644 index 3c53609b84e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMover.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.file.CopyOption; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.HashSet; -import java.util.Set; -import java.util.logging.Logger; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Utility for idempotent move of (any type of) file. - * - * @author hakonhall - */ -public class FileMover { - private static final Logger logger = Logger.getLogger(FileMover.class.getName()); - - private final Path source; - private final Path destination; - private final Set<CopyOption> moveOptions = new HashSet<>(); - - public FileMover(Path source, Path destination) { - this.source = source; - this.destination = destination; - } - - public FileMover replaceExisting() { - moveOptions.add(StandardCopyOption.REPLACE_EXISTING); - return this; - } - - public FileMover atomic() { - moveOptions.add(StandardCopyOption.ATOMIC_MOVE); - return this; - } - - /** - * Move file. - * - * @return false if the source doesn't exist while the destination do. - * @see Files#move(Path, Path, CopyOption...) Files.move() - */ - public boolean converge(TaskContext context) { - if (!Files.exists(source) && Files.exists(destination)) return false; - uncheck(() -> Files.move(source, destination, moveOptions.toArray(CopyOption[]::new))); - context.recordSystemModification(logger, "Moved " + source + " to " + destination); - return true; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java deleted file mode 100644 index b466b878ce5..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.Optional; - -/** - * A snapshot of the attributes of the file for a given path, and file content if it is a regular file. - * - * @author hakonhall - */ -public class FileSnapshot { - private final Path path; - private final Optional<FileAttributes> attributes; - private final Optional<byte[]> content; - - public static FileSnapshot forPath(Path path) { return forNonExistingFile(path).snapshot(); } - - /** Guaranteed to not throw any exceptions. */ - public static FileSnapshot forNonExistingFile(Path path) { - return new FileSnapshot(path, Optional.empty(), Optional.empty()); - } - - private static FileSnapshot forRegularFile(Path path, FileAttributes attributes, byte[] content) { - if (!attributes.isRegularFile()) throw new IllegalArgumentException(path + " is not a regular file"); - return new FileSnapshot(path, Optional.of(attributes), Optional.of(content)); - } - - private static FileSnapshot forOtherFile(Path path, FileAttributes attributes) { - if (attributes.isRegularFile()) throw new IllegalArgumentException(path + " is a regular file"); - return new FileSnapshot(path, Optional.of(attributes), Optional.empty()); - } - - private FileSnapshot(Path path, Optional<FileAttributes> attributes, Optional<byte[]> content) { - this.path = path; - this.attributes = attributes; - this.content = content; - } - - public Path path() { return path; } - - /** Whether there was a file (or directory) at path. */ - public boolean exists() { return attributes.isPresent(); } - - /** Returns the file attributes if the file exists. */ - public Optional<FileAttributes> attributes() { return attributes; } - - /** Returns the file content if the file exists and is a regular file. */ - public Optional<byte[]> content() { return content; } - - /** Returns the file UTF-8 content if it exists and is a regular file. */ - public Optional<String> utf8Content() { return content.map(c -> new String(c, StandardCharsets.UTF_8)); } - - /** Returns an up-to-date snapshot of the path, possibly {@code this} if last modified time has not changed. */ - public FileSnapshot snapshot() { - Optional<FileAttributes> currentAttributes = new UnixPath(path).getAttributesIfExists(); - if (currentAttributes.isPresent()) { - - // 'this' may still be valid, depending on last modified times. - if (attributes.isPresent()) { - Instant previousModifiedTime = attributes.get().lastModifiedTime(); - Instant currentModifiedTime = currentAttributes.get().lastModifiedTime(); - if (currentModifiedTime.compareTo(previousModifiedTime) <= 0) { - return this; - } - } - - if (currentAttributes.get().isRegularFile()) { - Optional<byte[]> content = IOExceptionUtil.ifExists(() -> Files.readAllBytes(path)); - return content.map(bytes -> FileSnapshot.forRegularFile(path, currentAttributes.get(), bytes)) - // File was removed after getting attributes and before getting content. - .orElseGet(() -> FileSnapshot.forNonExistingFile(path)); - } else { - return FileSnapshot.forOtherFile(path, currentAttributes.get()); - } - } else { - return attributes.isPresent() ? FileSnapshot.forNonExistingFile(path) : this /* avoid allocation */; - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java deleted file mode 100644 index bc572ce82a9..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.file.Path; -import java.time.Instant; -import java.util.Arrays; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * Class to minimize resource usage with repetitive and mostly identical, idempotent, and - * mutating file operations, e.g. setting file content, setting owner, etc. - * - * Only changes to the file is logged. - * - * @author hakohall - */ -// @ThreadUnsafe -public class FileSync { - private static final Logger logger = Logger.getLogger(FileSync.class.getName()); - - private final UnixPath path; - private final FileContentCache contentCache; - private final FileAttributesCache attributesCache; - - public FileSync(Path path) { - this.path = new UnixPath(path); - this.contentCache = new FileContentCache(this.path); - this.attributesCache = new FileAttributesCache(this.path); - } - - public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData) { - return convergeTo(taskContext, partialFileData, false); - } - - /** - * CPU, I/O, and memory usage is optimized for repeated calls with the same arguments. - * - * @param atomicWrite Whether to write updates to a temporary file in the same directory, and atomically move it - * to path. Ensures the file cannot be read while in the middle of writing it. - * @return true if the system was modified: content was written, or owner was set, etc. - * system is only modified if necessary (different). - */ - public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData, boolean atomicWrite) { - boolean modifiedSystem = false; - - if (partialFileData.getContent().isPresent()) { - modifiedSystem |= convergeTo(taskContext, partialFileData.getContent().get(), atomicWrite, partialFileData.getPermissions()); - } - - AttributeSync attributeSync = new AttributeSync(path.toPath()).with(partialFileData); - modifiedSystem |= attributeSync.converge(taskContext, this.attributesCache); - - return modifiedSystem; - } - - /** - * CPU, I/O, and memory usage is optimized for repeated calls with the same argument. - * - * @param atomicWrite Whether to write updates to a temporary file in the same directory, and atomically move it - * to path. Ensures the file cannot be read while in the middle of writing it. - * @param permissions Permissions if the file is created. - * @return true if the content was written. Only modified if necessary (different). - */ - public boolean convergeTo(TaskContext taskContext, byte[] content, boolean atomicWrite, Optional<String> permissions) { - Optional<Instant> lastModifiedTime = attributesCache.forceGet().map(FileAttributes::lastModifiedTime); - - if (lastModifiedTime.isEmpty()) { - taskContext.recordSystemModification(logger, "Creating file " + path + - permissions.map(p -> " with permissions " + p).orElse("")); - path.createParents(); - writeBytes(content, atomicWrite, permissions); - contentCache.updateWith(content, attributesCache.forceGet().orElseThrow().lastModifiedTime()); - return true; - } - - if (Arrays.equals(content, contentCache.get(attributesCache.getOrThrow().lastModifiedTime()))) { - return false; - } else { - taskContext.recordSystemModification(logger, "Patching file " + path); - // empty permissions here, because the file already exists and won't be applied anyway - writeBytes(content, atomicWrite, Optional.empty()); - contentCache.updateWith(content, attributesCache.forceGet().orElseThrow().lastModifiedTime()); - return true; - } - } - - private void writeBytes(byte[] content, boolean atomic, Optional<String> permissions) { - if (atomic) { - UnixPath tmpPath = new UnixPath(path.toPath().getFileSystem().getPath(path.toPath().toString() + ".FileSyncTmp")); - if (permissions.isPresent()) { - tmpPath.writeBytes(content, permissions.get()); - } else { - tmpPath.writeBytes(content); - } - tmpPath.atomicMove(path.toPath()); - } else { - if (permissions.isPresent()) { - path.writeBytes(content, permissions.get()); - } else { - path.writeBytes(content); - } - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java deleted file mode 100644 index aa6364f2a98..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; -import java.util.function.Supplier; - -/** - * Write a file - * - * @author hakonhall - */ -public class FileWriter { - private final Path path; - private final FileSync fileSync; - private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder(); - private final Optional<ByteArraySupplier> contentProducer; - - private boolean atomicWrite = false; - private boolean overwriteExistingFile = true; - - public FileWriter(Path path) { - this(path, Optional.empty()); - } - - public FileWriter(Path path, Supplier<String> contentProducer) { - this(path, () -> contentProducer.get().getBytes(StandardCharsets.UTF_8)); - } - - public FileWriter(Path path, ByteArraySupplier contentProducer) { - this(path, Optional.of(contentProducer)); - } - - private FileWriter(Path path, Optional<ByteArraySupplier> contentProducer) { - this.path = path; - this.fileSync = new FileSync(path); - this.contentProducer = contentProducer; - } - - public Path path() { return path; } - - public FileWriter withOwnerId(int ownerId) { - fileDataBuilder.withOwnerId(ownerId); - return this; - } - - public FileWriter withGroupId(int groupId) { - fileDataBuilder.withGroupId(groupId); - return this; - } - - /** @see UnixPath#setPermissions */ - public FileWriter withPermissions(String permissions) { - fileDataBuilder.withPermissions(permissions); - return this; - } - - public FileWriter atomicWrite(boolean atomicWrite) { - this.atomicWrite = atomicWrite; - return this; - } - - public FileWriter onlyIfFileDoesNotAlreadyExist() { - overwriteExistingFile = false; - return this; - } - - public boolean converge(TaskContext context) { - return converge(context, contentProducer.orElseThrow().get()); - } - - public boolean converge(TaskContext context, String utf8Content) { - return converge(context, utf8Content.getBytes(StandardCharsets.UTF_8)); - } - - public boolean converge(TaskContext context, byte[] content) { - if (!overwriteExistingFile && Files.isRegularFile(path)) { - return false; - } - - fileDataBuilder.withContent(content); - PartialFileData fileData = fileDataBuilder.create(); - return fileSync.convergeTo(context, fileData, atomicWrite); - } - - @FunctionalInterface - public interface ByteArraySupplier extends Supplier<byte[]> { } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java deleted file mode 100644 index a0db5a3cb16..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.yolean.Exceptions; - -import java.io.UncheckedIOException; -import java.nio.file.NoSuchFileException; -import java.util.Optional; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Utils related to IOException. - * - * @author hakonhall - */ -public class IOExceptionUtil { - /** - * Useful if it's not known whether a file or directory exists, in case e.g. - * NoSuchFileException is thrown and the caller wants an Optional.empty() in that case. - */ - public static <T> Optional<T> ifExists(Exceptions.SupplierThrowingIOException<T> supplier) { - try { - return Optional.ofNullable(uncheck(supplier)); - } catch (UncheckedIOException e) { - if (e.getCause() instanceof NoSuchFileException) { - return Optional.empty(); - } - - throw e; - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java deleted file mode 100644 index 88b403ba443..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit.Type.REPLACE; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit.Type.NONE; - -/** - * @author hakonhall - */ -public class LineEdit { - enum Type { NONE, REPLACE } - - public static LineEdit none() { return insert(List.of(), List.of()); } - public static LineEdit remove() { return replaceWith(List.of()); } - - public static LineEdit insertBefore(String... prepend) { return insertBefore(List.of(prepend)); } - public static LineEdit insertBefore(List<String> prepend) { return insert(prepend, List.of()); } - public static LineEdit insertAfter(String... append) { return insertAfter(List.of(append)); } - public static LineEdit insertAfter(List<String> append) { return insert(List.of(), append); } - public static LineEdit insert(List<String> prepend, List<String> append) { return new LineEdit(NONE, prepend, append); } - - public static LineEdit replaceWith(String... lines) { return replaceWith(List.of(lines)); } - public static LineEdit replaceWith(List<String> insertLines) { return new LineEdit(REPLACE, List.of(), insertLines); } - - private final Type type; - private final List<String> prependLines; - private final List<String> appendLines; - - private LineEdit(Type type, List<String> prependLines, List<String> appendLines) { - this.type = type; - this.prependLines = List.copyOf(prependLines); - this.appendLines = List.copyOf(appendLines); - } - - public Type getType() { return type; } - public List<String> prependLines() { return prependLines; } - public List<String> appendLines() { return appendLines; } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEditor.java deleted file mode 100644 index a7dcb4dd32a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEditor.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.util.List; - -/** - * @author hakonhall - */ -public interface LineEditor { - /** - * @param line The line of a file. - * @return The edited line, or empty if the line should be removed. - */ - LineEdit edit(String line); - - /** - * Called after edit() has been called on all lines in the file. - * @return Lines to append to the file. - */ - List<String> onComplete(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java deleted file mode 100644 index 24c2ae8543d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.io.UncheckedIOException; -import java.nio.file.NotDirectoryException; -import java.nio.file.Path; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * Class to ensure a directory exists with the correct owner, group, and permissions. - * - * @author hakonhall - */ -public class MakeDirectory { - private static final Logger logger = Logger.getLogger(MakeDirectory.class.getName()); - - private final UnixPath path; - private final AttributeSync attributeSync; - private final FileAttributesCache attributesCache; - - private boolean createParents = false; - - public MakeDirectory(Path path) { - this.path = new UnixPath(path); - this.attributeSync = new AttributeSync(path); - this.attributesCache = new FileAttributesCache(this.path); - } - - /** - * Warning: The owner, group, and permissions of any created parent directories are NOT modified - */ - public MakeDirectory createParents() { this.createParents = true; return this; } - - public MakeDirectory withOwnerId(int ownerId) { attributeSync.withOwnerId(ownerId); return this; } - public MakeDirectory withGroupId(int groupId) { attributeSync.withGroupId(groupId); return this; } - public MakeDirectory withPermissions(String permissions) { - attributeSync.withPermissions(permissions); - return this; - } - - public boolean converge(TaskContext context) { - boolean systemModified = false; - - Optional<FileAttributes> attributes = attributesCache.forceGet(); - if (attributes.isPresent()) { - if (!attributes.get().isDirectory()) { - throw new UncheckedIOException(new NotDirectoryException(path.toString())); - } - } else { - Optional<String> permissions = attributeSync.getPermissions(); - if (createParents) { - // We'll skip logging system modification here, as we'll log about the creation - // of the directory next. - permissions.ifPresentOrElse(path::createParents, path::createParents); - } - - context.recordSystemModification(logger, "Creating directory " + path); - systemModified = true; - - permissions.ifPresentOrElse(path::createDirectory, path::createDirectory); - } - - systemModified |= attributeSync.converge(context, attributesCache); - - return systemModified; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java deleted file mode 100644 index b1d56b131bb..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/PartialFileData.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.Optional; - -/** - * Represents a subset of a file's content, owner, group, and permissions. - * - * @author hakonhall - */ -// @Immutable -public class PartialFileData { - private final Optional<byte[]> content; - private final Optional<Integer> ownerId; - private final Optional<Integer> groupId; - private final Optional<String> permissions; - - public static Builder builder() { - return new Builder(); - } - - private PartialFileData(Optional<byte[]> content, - Optional<Integer> ownerId, - Optional<Integer> groupId, - Optional<String> permissions) { - this.content = content; - this.ownerId = ownerId; - this.groupId = groupId; - this.permissions = permissions; - } - - public Optional<byte[]> getContent() { - return content; - } - - public Optional<Integer> getOwnerId() { - return ownerId; - } - - public Optional<Integer> getGroupId() { - return groupId; - } - - public Optional<String> getPermissions() { - return permissions; - } - - public static class Builder { - private Optional<byte[]> content = Optional.empty(); - private Optional<Integer> ownerId = Optional.empty(); - private Optional<Integer> groupId = Optional.empty(); - private Optional<String> permissions = Optional.empty(); - - public Builder withContent(byte[] content) { this.content = Optional.of(content); return this; } - public Builder withContent(String content, Charset charset) { return withContent(content.getBytes(charset)); } - public Builder withContent(String content) { return withContent(content, StandardCharsets.UTF_8); } - public Builder withOwnerId(int ownerId) { this.ownerId = Optional.of(ownerId); return this; } - public Builder withGroupId(int groupId) { this.groupId = Optional.of(groupId); return this; } - public Builder withPermissions(String permissions) { this.permissions = Optional.of(permissions); return this; } - - public PartialFileData create() { - return new PartialFileData(content, ownerId, groupId, permissions); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java deleted file mode 100644 index 50ca5db9d3d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBoolean.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.file.Path; -import java.util.logging.Logger; - -/** - * Class wrapping a boolean stored on disk. - * - * <p>The implementation is compatible with {@link StoredInteger} when absence or 0 means false. - * - * @author hakonhall - */ -public class StoredBoolean { - private static final Logger logger = Logger.getLogger(StoredBoolean.class.getName()); - - private final UnixPath path; - - /** The parent directory must exist. Value is false by default. */ - public StoredBoolean(Path path) { - this.path = new UnixPath(path); - } - - public boolean value() { - return path.readUtf8FileIfExists().map(String::trim).map(s -> !"0".equals(s)).orElse(false); - } - - /** Sets value to true. */ - public void set(TaskContext context) { - if (!value()) { - context.log(logger, "Writes " + path); - path.writeUtf8File("1"); - } - } - - public void set(TaskContext context, boolean value) { - if (value) { - set(context); - } else { - clear(context); - } - } - - /** Sets value to false. */ - public void clear(TaskContext context) { - if (value()) { - context.log(logger, "Deletes " + path); - path.deleteIfExists(); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java deleted file mode 100644 index 19e8bcfcf93..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredDouble.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.file.Path; -import java.time.Instant; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.function.Supplier; -import java.util.logging.Logger; - -/** - * Class wrapping a float stored on disk - * - * @author freva - */ -public class StoredDouble implements Supplier<OptionalDouble> { - - private static final Logger logger = Logger.getLogger(StoredDouble.class.getName()); - - private final UnixPath path; - - public StoredDouble(Path path) { - this.path = new UnixPath(path); - } - - @Override - public OptionalDouble get() { - return path.readUtf8FileIfExists().stream().mapToDouble(Double::parseDouble).findAny(); - } - - public void write(TaskContext taskContext, double value) { - path.writeUtf8File(Double.toString(value)); - taskContext.log(logger, "Stored new double in %s: %f", path, value); - } - - public void clear() { - path.deleteIfExists(); - } - - public Optional<Instant> getLastModifiedTime() { - return path.getAttributesIfExists().map(FileAttributes::lastModifiedTime); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java deleted file mode 100644 index ec4d64db0e3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.file.Path; -import java.time.Instant; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.function.Supplier; -import java.util.logging.Logger; - -/** - * Class wrapping an integer stored on disk - * - * @author freva - */ -public class StoredInteger implements Supplier<OptionalInt> { - - private static final Logger logger = Logger.getLogger(StoredInteger.class.getName()); - - private final UnixPath path; - - public StoredInteger(Path path) { - this.path = new UnixPath(path); - } - - @Override - public OptionalInt get() { - return path.readUtf8FileIfExists().stream().mapToInt(Integer::parseInt).findAny(); - } - - public void write(TaskContext taskContext, int value) { - path.writeUtf8File(Integer.toString(value)); - taskContext.log(logger, "Stored new integer in %s: %d", path, value); - } - - public Optional<Instant> getLastModifiedTime() { - return path.getAttributesIfExists().map(FileAttributes::lastModifiedTime); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java deleted file mode 100644 index 2436ba306ac..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.Velocity; -import org.apache.velocity.runtime.RuntimeConstants; -import org.slf4j.helpers.NOPLogger; - -import java.io.StringWriter; -import java.nio.file.Files; -import java.nio.file.Path; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Uses the Velocity engine to render a template, to and from both String and Path objects. - * - * @author hakonhall - * @author jonmv - */ -public class Template { - - static { - Velocity.addProperty(RuntimeConstants.RUNTIME_LOG_INSTANCE, NOPLogger.NOP_LOGGER); - Velocity.init(); - } - - private final VelocityContext velocityContext = new VelocityContext(); - private final String template; - - private Template(String template) { - this.template = template; - } - - public static Template at(Path templatePath) { - return of(uncheck(() -> new String(Files.readAllBytes(templatePath)))); - } - - public static Template of(String template) { - return new Template(template); - } - - public Template set(String name, Object value) { - velocityContext.put(name, value); - return this; - } - - public FileWriter getFileWriterTo(Path destinationPath) { - return new FileWriter(destinationPath, this::render); - } - - public String render() { - StringWriter writer = new StringWriter(); - Velocity.evaluate(velocityContext, writer, "Template", template); - return writer.toString(); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java deleted file mode 100644 index 1983e94e6f5..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.nio.channels.SeekableByteChannel; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.NotDirectoryException; -import java.nio.file.OpenOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.FileTime; -import java.nio.file.attribute.GroupPrincipal; -import java.nio.file.attribute.PosixFileAttributeView; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.nio.file.attribute.UserPrincipal; -import java.nio.file.attribute.UserPrincipalLookupService; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists; -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Thin wrapper around java.nio.file.Path, especially nice for UNIX-specific features. - * - * @author hakonhall - */ -// @Immutable -public class UnixPath { - - private static final Set<OpenOption> DEFAULT_OPEN_OPTIONS = - Set.of(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); - - private final Path path; - - public UnixPath(Path path) { this.path = path; } - public UnixPath(String path) { this(Path.of(path)); } - - public Path toPath() { return path; } - public UnixPath resolve(String relativeOrAbsolutePath) { return new UnixPath(path.resolve(relativeOrAbsolutePath)); } - - public UnixPath getParent() { - Path parentPath = path.getParent(); - if (parentPath == null) { - throw new IllegalStateException("Path has no parent directory: '" + path + "'"); - } - - return new UnixPath(parentPath); - } - - public String getFilename() { - Path filename = path.getFileName(); - if (filename == null) { - // E.g. "/". - throw new IllegalStateException("Path has no filename: '" + path + "'"); - } - - return filename.toString(); - } - - public boolean exists() { return Files.exists(path); } - - public String readUtf8File() { - return new String(readBytes(), StandardCharsets.UTF_8); - } - - public Optional<String> readUtf8FileIfExists() { - try { - return Optional.of(Files.readString(path)); - } catch (NoSuchFileException ignored) { - return Optional.empty(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public byte[] readBytes() { - return uncheck(() -> Files.readAllBytes(path)); - } - - /** Reads and returns all bytes contained in this path, if any such path exists. */ - public Optional<byte[]> readBytesIfExists() { - try { - return Optional.of(Files.readAllBytes(path)); - } catch (NoSuchFileException ignored) { - return Optional.empty(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public List<String> readLines() { - return uncheck(() -> Files.readAllLines(path)); - } - - /** Create an empty file and return true, or false if the file already exists (the file may not be regular). */ - public boolean create() { - try { - Files.createFile(path); - return true; - } catch (FileAlreadyExistsException ignored) { - return false; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public UnixPath writeUtf8File(String content, OpenOption... options) { - return writeBytes(content.getBytes(StandardCharsets.UTF_8), options); - } - - public UnixPath writeUtf8File(String content, String permissions, OpenOption... options) { - return writeBytes(content.getBytes(StandardCharsets.UTF_8), permissions, options); - } - - public UnixPath writeBytes(byte[] content, OpenOption... options) { - return writeBytes(content, null, options); - } - - public UnixPath writeBytes(byte[] content, String permissions, OpenOption... options) { - FileAttribute<?>[] attributes = Optional.ofNullable(permissions) - .map(this::permissionsAsFileAttributes) - .orElseGet(() -> new FileAttribute<?>[0]); - - Set<OpenOption> optionsSet = options.length == 0 ? DEFAULT_OPEN_OPTIONS : Set.of(options); - - try (SeekableByteChannel channel = Files.newByteChannel(path, optionsSet, attributes)) { - channel.write(ByteBuffer.wrap(content)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return this; - } - - /** Write a file to the same dir as this, and then atomically move it to this' path. */ - public UnixPath atomicWriteBytes(byte[] content) { - UnixPath temporaryPath = getParent().resolve(getFilename() + ".10Ia2f4N5"); - temporaryPath.writeBytes(content); - temporaryPath.atomicMove(path); - return this; - } - - public String getPermissions() { - return getAttributes().permissions(); - } - - /** - * @param permissions Example: "rwxr-x---" means rwx for owner, rx for group, - * and no permissions for others. - */ - public UnixPath setPermissions(String permissions) { - Set<PosixFilePermission> permissionSet = getPosixFilePermissionsFromString(permissions); - uncheck(() -> Files.setPosixFilePermissions(path, permissionSet)); - return this; - } - - public int getOwnerId() { - return getAttributes().ownerId(); - } - - public UnixPath setOwner(String user) { return setOwner(user, "user"); } - public UnixPath setOwnerId(int uid) { return setOwner(String.valueOf(uid), "uid"); } - private UnixPath setOwner(String owner, String type) { - UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService(); - UserPrincipal principal = uncheck( - () -> service.lookupPrincipalByName(owner), - "While looking up %s %s", type, owner); - uncheck(() -> Files.setOwner(path, principal)); - return this; - } - - public int getGroupId() { - return getAttributes().groupId(); - } - - public UnixPath setGroup(String group) { return setGroup(group, "group"); } - public UnixPath setGroupId(int gid) { return setGroup(String.valueOf(gid), "gid"); } - private UnixPath setGroup(String group, String type) { - UserPrincipalLookupService service = path.getFileSystem().getUserPrincipalLookupService(); - GroupPrincipal principal = uncheck( - () -> service.lookupPrincipalByGroupName(group), - "While looking up group %s %s", type, group); - uncheck(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class).setGroup(principal)); - return this; - } - - public Instant getLastModifiedTime() { - return getAttributes().lastModifiedTime(); - } - - public UnixPath updateLastModifiedTime() { - return setLastModifiedTime(Instant.now()); - } - - public UnixPath setLastModifiedTime(Instant instant) { - uncheck(() -> Files.setLastModifiedTime(path, FileTime.from(instant))); - return this; - } - - public FileAttributes getAttributes() { - return uncheck(() -> FileAttributes.fromAttributes(Files.readAttributes(path, "unix:*"))); - } - - public Optional<FileAttributes> getAttributesIfExists() { - return ifExists(this::getAttributes); - } - - public UnixPath createNewFile(String... permissions) { - uncheck(() -> Files.createFile(path, permissionsAsFileAttributes(permissions))); - return this; - } - - public UnixPath createParents(String... permissions) { - getParent().createDirectories(permissions); - return this; - } - - /** Create directory with given permissions and return true, or false if it already exists. */ - public boolean createDirectory(String... permissions) { - try { - Files.createDirectory(path, permissionsAsFileAttributes(permissions)); - } catch (FileAlreadyExistsException ignore) { - return false; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return true; - } - - public UnixPath createDirectories(String... permissions) { - uncheck(() -> Files.createDirectories(path, permissionsAsFileAttributes(permissions))); - return this; - } - - /** - * Returns whether this path is a directory. Symlinks are followed, so this returns true for symlinks pointing to a - * directory. - */ - public boolean isDirectory() { - return uncheck(() -> Files.isDirectory(path)); - } - - /** Returns whether this is a symlink */ - public boolean isSymbolicLink() { - return Files.isSymbolicLink(path); - } - - /** - * Similar to rm -rf file: - * - It's not an error if file doesn't exist - * - If file is a directory, it and all content is removed - * - For symlinks: Only the symlink is removed, not what the symlink points to - */ - public boolean deleteRecursively() { - if (!isSymbolicLink() && isDirectory()) { - try (Stream<UnixPath> paths = listContentsOfDirectory()) { - paths.forEach(UnixPath::deleteRecursively); - } - } - return uncheck(() -> Files.deleteIfExists(path)); - } - - public boolean deleteIfExists() { - return uncheck(() -> Files.deleteIfExists(path)); - } - - /** @return false path does not exist, is not a directory, or has at least one entry. */ - public boolean isEmptyDirectory() { - try (var entryStream = Files.list(path)) { - return entryStream.findAny().isEmpty(); - } catch (NotDirectoryException | NoSuchFileException e) { - return false; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - /** Lists the contents of this as a stream. Callers should use try-with to ensure that the stream is closed */ - public Stream<UnixPath> listContentsOfDirectory() { - try { - // Avoid the temptation to collect the stream here as collecting a directory with a high number of entries - // can quickly lead to out of memory conditions - return Files.list(path).map(UnixPath::new); - } catch (NoSuchFileException ignored) { - return Stream.empty(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to list contents of directory " + path.toAbsolutePath(), e); - } - } - - /** This path must be on the same file system as the to-path. Returns UnixPath of 'to'. */ - public UnixPath atomicMove(Path to) { - uncheck(() -> Files.move(path, to, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING)); - return new UnixPath(to); - } - - public boolean moveIfExists(Path to) { - try { - Files.move(path, to); - return true; - } catch (NoSuchFileException ignored) { - return false; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - /** - * Creates a symbolic link from {@code link} to {@code this} (the target) - * @param link the path for the symbolic link - * @return the path to the symbolic link - */ - public UnixPath createSymbolicLink(Path link) { - uncheck(() -> Files.createSymbolicLink(link, path)); - return new UnixPath(link); - } - - @Override - public String toString() { - return path.toString(); - } - - private FileAttribute<?>[] permissionsAsFileAttributes(String... permissions) { - if (permissions.length == 0) return new FileAttribute<?>[0]; - if (permissions.length > 1) - throw new IllegalArgumentException("Expected permissions to not be set or be a single string"); - - return new FileAttribute<?>[]{PosixFilePermissions.asFileAttribute(getPosixFilePermissionsFromString(permissions[0]))}; - } - - private Set<PosixFilePermission> getPosixFilePermissionsFromString(String permissions) { - try { - return PosixFilePermissions.fromString(permissions); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Failed to set permissions '" + - permissions + "' on path " + path, e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java deleted file mode 100644 index 93ad0f21fe0..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.util.Objects; - -/** - * A regular UNIX-style user and its primary group. - * - * @author mpolden - */ -public class UnixUser { - - public static final UnixUser ROOT = new UnixUser("root", 0, "root", 0); - public static final UnixUser VESPA = new UnixUser("vespa", 1000, "vespa", 1000); - - private final String name; - private final int uid; - private final String group; - private final int gid; - - public UnixUser(String name, int uid, String group, int gid) { - this.name = name; - this.uid = uid; - this.group = group; - this.gid = gid; - } - - /** Username of this */ - public String name() { return name; } - - /** User ID of this */ - public int uid() { return uid; } - - /** Primary group of this */ - public String group() { return group; } - - /** Primary group ID of this */ - public int gid() { return gid; } - - @Override - public String toString() { - return "user " + name + ":" + group; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - UnixUser unixUser = (UnixUser) o; - return uid == unixUser.uid && name.equals(unixUser.name) && - gid == unixUser.gid && group.equals(unixUser.group); - } - - @Override - public int hashCode() { - return Objects.hash(uid, name, gid, group); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/package-info.java deleted file mode 100644 index a15b918913a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java deleted file mode 100644 index 77978e65f42..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerAttributeViews.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import java.io.IOException; -import java.nio.file.ProviderMismatchException; -import java.nio.file.attribute.FileTime; -import java.nio.file.attribute.GroupPrincipal; -import java.nio.file.attribute.PosixFileAttributeView; -import java.nio.file.attribute.PosixFileAttributes; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.UserPrincipal; -import java.util.Map; -import java.util.Set; - -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal; -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal; - -/** - * @author freva - */ -class ContainerAttributeViews { - - static class ContainerPosixFileAttributeView implements PosixFileAttributeView { - private final PosixFileAttributeView posixFileAttributeView; - private final ContainerPosixFileAttributes fileAttributes; - - ContainerPosixFileAttributeView(PosixFileAttributeView posixFileAttributeView, - ContainerPosixFileAttributes fileAttributes) { - this.posixFileAttributeView = posixFileAttributeView; - this.fileAttributes = fileAttributes; - } - - @Override public String name() { return "posix"; } - @Override public UserPrincipal getOwner() { return fileAttributes.owner(); } - @Override public PosixFileAttributes readAttributes() { return fileAttributes; } - - @Override - public void setOwner(UserPrincipal owner) throws IOException { - if (!(owner instanceof ContainerUserPrincipal)) throw new ProviderMismatchException(); - posixFileAttributeView.setOwner(((ContainerUserPrincipal) owner).baseFsPrincipal()); - } - - @Override - public void setGroup(GroupPrincipal group) throws IOException { - if (!(group instanceof ContainerGroupPrincipal)) throw new ProviderMismatchException(); - posixFileAttributeView.setGroup(((ContainerGroupPrincipal) group).baseFsPrincipal()); - } - - @Override - public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) throws IOException { - posixFileAttributeView.setTimes(lastModifiedTime, lastAccessTime, createTime); - } - - @Override - public void setPermissions(Set<PosixFilePermission> perms) throws IOException { - posixFileAttributeView.setPermissions(perms); - } - } - - static class ContainerPosixFileAttributes implements PosixFileAttributes { - private final Map<String, Object> attributes; - - ContainerPosixFileAttributes(Map<String, Object> attributes) { - this.attributes = attributes; - } - - @SuppressWarnings("unchecked") - @Override public Set<PosixFilePermission> permissions() { return (Set<PosixFilePermission>) attributes.get("permissions"); } - @Override public ContainerUserPrincipal owner() { return (ContainerUserPrincipal) attributes.get("owner"); } - @Override public ContainerGroupPrincipal group() { return (ContainerGroupPrincipal) attributes.get("group"); } - @Override public FileTime lastModifiedTime() { return (FileTime) attributes.get("lastModifiedTime"); } - @Override public FileTime lastAccessTime() { return (FileTime) attributes.get("lastAccessTime"); } - @Override public FileTime creationTime() { return (FileTime) attributes.get("creationTime"); } - @Override public boolean isRegularFile() { return (boolean) attributes.get("isRegularFile"); } - @Override public boolean isDirectory() { return (boolean) attributes.get("isDirectory"); } - @Override public boolean isSymbolicLink() { return (boolean) attributes.get("isSymbolicLink"); } - @Override public boolean isOther() { return (boolean) attributes.get("isOther"); } - @Override public long size() { return (long) attributes.get("size"); } - @Override public Object fileKey() { return attributes.get("fileKey"); } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java deleted file mode 100644 index 3329a646671..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; - -import java.io.IOException; -import java.nio.file.FileStore; -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.nio.file.WatchService; -import java.util.Set; - -/** - * @author freva - */ -public class ContainerFileSystem extends FileSystem { - - private final ContainerFileSystemProvider containerFsProvider; - private final Path containerRootOnHost; - - ContainerFileSystem(ContainerFileSystemProvider containerFsProvider, Path containerRootOnHost) { - this.containerFsProvider = containerFsProvider; - this.containerRootOnHost = containerRootOnHost; - } - - public Path containerRootOnHost() { - return containerRootOnHost; - } - - public void createRoot() { - provider().createFileSystemRoot(); - } - - @Override - public ContainerFileSystemProvider provider() { - return containerFsProvider; - } - - @Override - public boolean isOpen() { - return true; - } - - @Override - public boolean isReadOnly() { - return false; - } - - @Override - public String getSeparator() { - return "/"; - } - - @Override - public Set<String> supportedFileAttributeViews() { - return Set.of("basic", "posix", "unix", "owner"); - } - - @Override - public ContainerUserPrincipalLookupService getUserPrincipalLookupService() { - return containerFsProvider.userPrincipalLookupService(); - } - - @Override - public ContainerPath getPath(String first, String... more) { - return ContainerPath.fromPathInContainer(this, Path.of(first, more), getUserPrincipalLookupService().userScope().root()); - } - - @Override - public void close() { - throw new UnsupportedOperationException(); - } - - @Override - public Iterable<Path> getRootDirectories() { - throw new UnsupportedOperationException(); - } - - @Override - public Iterable<FileStore> getFileStores() { - throw new UnsupportedOperationException(); - } - - @Override - public PathMatcher getPathMatcher(String syntaxAndPattern) { - throw new UnsupportedOperationException(); - } - - @Override - public WatchService newWatchService() { - throw new UnsupportedOperationException(); - } - - public static ContainerFileSystem create(Path containerStorageRoot, UserScope userScope) { - return new ContainerFileSystemProvider(containerStorageRoot, userScope).getFileSystem(null); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java deleted file mode 100644 index 469ddd89ea3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java +++ /dev/null @@ -1,348 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; - -import java.io.IOException; -import java.net.URI; -import java.nio.channels.SeekableByteChannel; -import java.nio.file.AccessMode; -import java.nio.file.CopyOption; -import java.nio.file.DirectoryStream; -import java.nio.file.FileStore; -import java.nio.file.FileSystem; -import java.nio.file.FileSystemAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.OpenOption; -import java.nio.file.Path; -import java.nio.file.ProviderMismatchException; -import java.nio.file.SecureDirectoryStream; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.FileAttributeView; -import java.nio.file.attribute.GroupPrincipal; -import java.nio.file.attribute.PosixFileAttributeView; -import java.nio.file.attribute.PosixFileAttributes; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.nio.file.attribute.UserPrincipal; -import java.nio.file.spi.FileSystemProvider; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributeView; -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerAttributeViews.ContainerPosixFileAttributes; -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal; -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal; -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author freva - */ -class ContainerFileSystemProvider extends FileSystemProvider { - - private static final FileAttribute<?> DEFAULT_FILE_PERMISSIONS = PosixFilePermissions.asFileAttribute(Set.of( // 0640 - PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.GROUP_READ)); - private static final FileAttribute<?> DEFAULT_DIRECTORY_PERMISSIONS = PosixFilePermissions.asFileAttribute(Set.of( // 0750 - PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_EXECUTE)); - - private final ContainerFileSystem containerFs; - private final ContainerUserPrincipalLookupService userPrincipalLookupService; - - ContainerFileSystemProvider(Path containerRootOnHost, UserScope userScope) { - this.containerFs = new ContainerFileSystem(this, containerRootOnHost); - this.userPrincipalLookupService = new ContainerUserPrincipalLookupService( - containerRootOnHost.getFileSystem().getUserPrincipalLookupService(), userScope); - } - - public ContainerUserPrincipalLookupService userPrincipalLookupService() { - return userPrincipalLookupService; - } - - @Override - public String getScheme() { - return "file"; - } - - @Override - public FileSystem newFileSystem(URI uri, Map<String, ?> env) { - throw new FileSystemAlreadyExistsException(); - } - - @Override - public ContainerFileSystem getFileSystem(URI uri) { - return containerFs; - } - - @Override - public Path getPath(URI uri) { - throw new UnsupportedOperationException(); - } - - @Override - public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException { - Path pathOnHost = pathOnHost(path); - try (SecureDirectoryStream<Path> sds = leafDirectoryStream(pathOnHost)) { - boolean existedBefore = Files.exists(pathOnHost); - SeekableByteChannel seekableByteChannel = sds.newByteChannel( - pathOnHost.getFileName(), addNoFollow(options), addPermissions(DEFAULT_FILE_PERMISSIONS, attrs)); - if (!existedBefore) fixOwnerToContainerRoot(toContainerPath(path)); - return seekableByteChannel; - } - } - - @Override - public DirectoryStream<Path> newDirectoryStream(Path dir, DirectoryStream.Filter<? super Path> filter) throws IOException { - Path pathOnHost = pathOnHost(dir); - return new ContainerDirectoryStream(provider(pathOnHost).newDirectoryStream(pathOnHost, filter), - toContainerPath(dir).user()); - } - - @Override - public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException { - Path pathOnHost = pathOnHost(dir); - boolean existedBefore = Files.exists(pathOnHost); - provider(pathOnHost).createDirectory(pathOnHost, addPermissions(DEFAULT_DIRECTORY_PERMISSIONS, attrs)); - if (!existedBefore) fixOwnerToContainerRoot(toContainerPath(dir)); - } - - @Override - public void delete(Path path) throws IOException { - Path pathOnHost = pathOnHost(path); - provider(pathOnHost).delete(pathOnHost); - } - - @Override - public void copy(Path source, Path target, CopyOption... options) throws IOException { - // Only called when both 'source' and 'target' have 'this' as the FS provider - Path targetPathOnHost = pathOnHost(target); - provider(targetPathOnHost).copy(pathOnHost(source), targetPathOnHost, addNoFollow(options)); - } - - @Override - public void move(Path source, Path target, CopyOption... options) throws IOException { - // Only called when both 'source' and 'target' have 'this' as the FS provider - Path targetPathOnHost = pathOnHost(target); - provider(targetPathOnHost).move(pathOnHost(source), targetPathOnHost, addNoFollow(options)); - } - - @Override - public void createSymbolicLink(Path link, Path target, FileAttribute<?>... attrs) throws IOException { - Path pathOnHost = pathOnHost(link); - boolean existedBefore = Files.exists(pathOnHost, LinkOption.NOFOLLOW_LINKS); - if (target instanceof ContainerPath) - target = pathOnHost.getFileSystem().getPath(toContainerPath(target).pathInContainer()); - provider(pathOnHost).createSymbolicLink(pathOnHost, target, attrs); - if (!existedBefore) fixOwnerToContainerRoot(toContainerPath(link)); - } - - @Override - public Path readSymbolicLink(Path link) throws IOException { - Path pathOnHost = pathOnHost(link); - return provider(pathOnHost).readSymbolicLink(pathOnHost); - } - - @Override - public boolean isSameFile(Path path, Path path2) throws IOException { - // 'path' FS provider should be 'this' - if (path2 instanceof ContainerPath) - path2 = pathOnHost(path2); - Path pathOnHost = pathOnHost(path); - return provider(pathOnHost).isSameFile(pathOnHost, path2); - } - - @Override - public boolean isHidden(Path path) throws IOException { - Path pathOnHost = pathOnHost(path); - return provider(pathOnHost).isHidden(pathOnHost); - } - - @Override - public FileStore getFileStore(Path path) { - throw new UnsupportedOperationException(); - } - - @Override - public void checkAccess(Path path, AccessMode... modes) throws IOException { - Path pathOnHost = pathOnHost(path); - provider(pathOnHost).checkAccess(pathOnHost, modes); - } - - @Override - @SuppressWarnings("unchecked") - public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) { - if (!type.isAssignableFrom(PosixFileAttributeView.class)) return null; - Path pathOnHost = pathOnHost(path); - FileSystemProvider provider = pathOnHost.getFileSystem().provider(); - if (type == BasicFileAttributeView.class) // Basic view doesn't have owner/group fields, forward to base FS provider - return provider.getFileAttributeView(pathOnHost, type, addNoFollow(options)); - - PosixFileAttributeView view = provider.getFileAttributeView(pathOnHost, PosixFileAttributeView.class, addNoFollow(options)); - return (V) new ContainerPosixFileAttributeView(view, - uncheck(() -> new ContainerPosixFileAttributes(readAttributes(path, "unix:*", addNoFollow(options))))); - } - - @Override - @SuppressWarnings("unchecked") - public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException { - if (!type.isAssignableFrom(PosixFileAttributes.class)) throw new UnsupportedOperationException(); - Path pathOnHost = pathOnHost(path); - if (type == BasicFileAttributes.class) - return pathOnHost.getFileSystem().provider().readAttributes(pathOnHost, type, addNoFollow(options)); - - // Non-basic requests need to be upgraded to unix:* to get owner,group,uid,gid fields, which are then re-mapped - return (A) new ContainerPosixFileAttributes(readAttributes(path, "unix:*", addNoFollow(options))); - } - - @Override - public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException { - Path pathOnHost = pathOnHost(path); - int index = attributes.indexOf(':'); - if (index < 0 || attributes.startsWith("basic:")) - return provider(pathOnHost).readAttributes(pathOnHost, attributes, addNoFollow(options)); - - Map<String, Object> attrs = new HashMap<>(provider(pathOnHost).readAttributes(pathOnHost, "unix:*", addNoFollow(options))); - int uid = userPrincipalLookupService.userIdInContainer((int) attrs.get("uid")); - int gid = userPrincipalLookupService.groupIdInContainer((int) attrs.get("gid")); - attrs.put("uid", uid); - attrs.put("gid", gid); - attrs.put("owner", userPrincipalLookupService.userPrincipal(uid, (UserPrincipal) attrs.get("owner"))); - attrs.put("group", userPrincipalLookupService.groupPrincipal(gid, (GroupPrincipal) attrs.get("group"))); - return attrs; - } - - @Override - public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException { - Path pathOnHost = pathOnHost(path); - provider(pathOnHost).setAttribute(pathOnHost, attribute, fixAttributeValue(attribute, value), addNoFollow(options)); - } - - private Object fixAttributeValue(String attribute, Object value) { - int index = attribute.indexOf(':'); - if (index > 0) { - switch (attribute.substring(index + 1)) { - case "owner": return cast(value, ContainerUserPrincipal.class).baseFsPrincipal(); - case "group": return cast(value, ContainerGroupPrincipal.class).baseFsPrincipal(); - case "uid": return userPrincipalLookupService.userIdOnHost(cast(value, Integer.class)); - case "gid": return userPrincipalLookupService.groupIdOnHost(cast(value, Integer.class)); - } - } // else basic file attribute - return value; - } - - void createFileSystemRoot() { - ContainerPath root = containerFs.getPath("/"); - if (!Files.exists(root)) { - uncheck(() -> { - Files.createDirectories(root.pathOnHost()); - fixOwnerToContainerRoot(root); - }); - } - } - - private void fixOwnerToContainerRoot(ContainerPath path) throws IOException { - setAttribute(path, "unix:uid", path.user().uid(), LinkOption.NOFOLLOW_LINKS); - setAttribute(path, "unix:gid", path.user().gid(), LinkOption.NOFOLLOW_LINKS); - } - - private SecureDirectoryStream<Path> leafDirectoryStream(Path pathOnHost) throws IOException { - Path containerRoot = containerFs.containerRootOnHost(); - SecureDirectoryStream<Path> sds = ((SecureDirectoryStream<Path>) Files.newDirectoryStream(containerRoot)); - for (int i = containerRoot.getNameCount(); i < pathOnHost.getNameCount() - 1; i++) { - SecureDirectoryStream<Path> next = sds.newDirectoryStream(pathOnHost.getName(i), LinkOption.NOFOLLOW_LINKS); - sds.close(); - sds = next; - } - return sds; - } - - private class ContainerDirectoryStream implements DirectoryStream<Path> { - private final DirectoryStream<Path> hostDirectoryStream; - private final UnixUser user; - - private ContainerDirectoryStream(DirectoryStream<Path> hostDirectoryStream, UnixUser user) { - this.hostDirectoryStream = hostDirectoryStream; - this.user = user; - } - - @Override - public Iterator<Path> iterator() { - Iterator<Path> hostPathIterator = hostDirectoryStream.iterator(); - return new Iterator<>() { - @Override - public boolean hasNext() { - return hostPathIterator.hasNext(); - } - - @Override - public Path next() { - Path pathOnHost = hostPathIterator.next(); - return ContainerPath.fromPathOnHost(containerFs, pathOnHost, user); - } - }; - } - - @Override - public void close() throws IOException { - hostDirectoryStream.close(); - } - } - - static ContainerPath toContainerPath(Path path) { - return cast(path, ContainerPath.class); - } - - private static <T> T cast(Object value, Class<T> type) { - if (type.isInstance(value)) return type.cast(value); - throw new ProviderMismatchException("Expected " + type.getSimpleName() + ", was " + value.getClass().getName()); - } - - private static Path pathOnHost(Path path) { - return toContainerPath(path).pathOnHost(); - } - - private static FileSystemProvider provider(Path path) { - return path.getFileSystem().provider(); - } - - private static Set<? extends OpenOption> addNoFollow(Set<? extends OpenOption> options) { - if (options.contains(LinkOption.NOFOLLOW_LINKS)) return options; - Set<OpenOption> copy = new HashSet<>(options); - copy.add(LinkOption.NOFOLLOW_LINKS); - return copy; - } - - private static LinkOption[] addNoFollow(LinkOption... options) { - if (Set.of(options).contains(LinkOption.NOFOLLOW_LINKS)) return options; - LinkOption[] copy = new LinkOption[options.length + 1]; - System.arraycopy(options, 0, copy, 0, options.length); - copy[options.length] = LinkOption.NOFOLLOW_LINKS; - return copy; - } - - private static CopyOption[] addNoFollow(CopyOption... options) { - if (Set.of(options).contains(LinkOption.NOFOLLOW_LINKS)) return options; - CopyOption[] copy = new CopyOption[options.length + 1]; - System.arraycopy(options, 0, copy, 0, options.length); - copy[options.length] = LinkOption.NOFOLLOW_LINKS; - return copy; - } - - private static FileAttribute<?>[] addPermissions(FileAttribute<?> defaultPermissions, FileAttribute<?>... attrs) { - for (FileAttribute<?> attr : attrs) { - if (attr.name().equals("posix:permissions") || attr.name().equals("unix:permissions")) - return attrs; - } - - FileAttribute<?>[] copy = new FileAttribute<?>[attrs.length + 1]; - System.arraycopy(attrs, 0, copy, 0, attrs.length); - copy[attrs.length] = defaultPermissions; - return copy; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java deleted file mode 100644 index 314e7cde5e2..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.WatchEvent; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystemProvider.toContainerPath; - -/** - * Represents a path in container that is mapped in from the host. ContainerPaths are always normalized and absolute. - * - * @author freva - */ -public class ContainerPath implements Path { - private final ContainerFileSystem containerFs; - private final Path pathOnHost; - private final String[] parts; - private final UnixUser user; - - private ContainerPath(ContainerFileSystem containerFs, Path pathOnHost, String[] parts, UnixUser user) { - this.containerFs = Objects.requireNonNull(containerFs); - this.pathOnHost = Objects.requireNonNull(pathOnHost); - this.parts = Objects.requireNonNull(parts); - this.user = Objects.requireNonNull(user); - - if (!pathOnHost.isAbsolute()) - throw new IllegalArgumentException("Path host must be absolute: " + pathOnHost); - Path containerRootOnHost = containerFs.containerRootOnHost(); - if (!pathOnHost.startsWith(containerRootOnHost)) - throw new IllegalArgumentException("Path on host (" + pathOnHost + ") must start with container root on host (" + containerRootOnHost + ")"); - } - - public Path pathOnHost() { return pathOnHost; } - public String pathInContainer() { return '/' + String.join("/", parts); } - public ContainerPath withUser(UnixUser user) { return new ContainerPath(containerFs, pathOnHost, parts, user); } - public UnixUser user() { return user; } - - @Override - public ContainerFileSystem getFileSystem() { - return containerFs; - } - - @Override - public ContainerPath getRoot() { - return resolve(containerFs, new String[0], Path.of("/"), user); - } - - @Override - public Path getFileName() { - if (parts.length == 0) return null; - return Path.of(parts[parts.length - 1]); - } - - @Override - public ContainerPath getParent() { - if (parts.length == 0) return null; - return new ContainerPath(containerFs, pathOnHost.getParent(), Arrays.copyOf(parts, parts.length-1), user); - } - - @Override - public int getNameCount() { - return parts.length; - } - - @Override - public Path getName(int index) { - return Path.of(parts[index]); - } - - @Override - public Path subpath(int beginIndex, int endIndex) { - if (beginIndex < 0 || beginIndex >= endIndex || endIndex > parts.length) - throw new IllegalArgumentException(); - if (endIndex - beginIndex == 1) return getName(beginIndex); - - String[] rest = new String[endIndex - beginIndex - 1]; - System.arraycopy(parts, beginIndex + 1, rest, 0, rest.length); - return Path.of(parts[beginIndex], rest); - } - - @Override public ContainerPath resolve(Path other) { return resolve(containerFs, parts, other, user); } - @Override public ContainerPath resolve(String other) { return resolve(Path.of(other)); } - @Override public ContainerPath resolveSibling(String other) { return resolve(Path.of("..", other)); } - - @Override - public boolean startsWith(Path other) { - if (other.getFileSystem() != containerFs) return false; - String[] otherParts = toContainerPath(other).parts; - if (parts.length < otherParts.length) return false; - - for (int i = 0; i < otherParts.length; i++) { - if ( ! parts[i].equals(otherParts[i])) return false; - } - return true; - } - - @Override - public boolean endsWith(Path other) { - int offset = parts.length - other.getNameCount(); - // If the other path is longer than this, or the other path is absolute and shorter than this - if (offset < 0 || (other.isAbsolute() && offset > 0)) return false; - - for (int i = 0; i < other.getNameCount(); i++) { - if ( ! parts[offset + i].equals(other.getName(i).toString())) return false; - } - return true; - } - - @Override - public boolean isAbsolute() { - // All container paths are normalized and absolute - return true; - } - - @Override - public ContainerPath normalize() { - // All container paths are normalized and absolute - return this; - } - - @Override - public ContainerPath toAbsolutePath() { - // All container paths are normalized and absolute - return this; - } - - @Override - public ContainerPath toRealPath(LinkOption... options) throws IOException { - Path realPathOnHost = pathOnHost.toRealPath(options); - if (realPathOnHost.equals(pathOnHost)) return this; - return fromPathOnHost(containerFs, realPathOnHost, user); - } - - @Override - public Path relativize(Path other) { - return pathOnHost.relativize(toContainerPath(other).pathOnHost); - } - - @Override - public URI toUri() { - throw new UnsupportedOperationException(); - } - - @Override - public WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) throws IOException { - return pathOnHost.register(watcher, events, modifiers); - } - - @Override - public int compareTo(Path other) { - return pathOnHost.compareTo(toContainerPath(other)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ContainerPath paths = (ContainerPath) o; - return containerFs.equals(paths.containerFs) && pathOnHost.equals(paths.pathOnHost) && Arrays.equals(parts, paths.parts); - } - - @Override - public int hashCode() { - int result = Objects.hash(containerFs, pathOnHost); - result = 31 * result + Arrays.hashCode(parts); - return result; - } - - @Override - public String toString() { - return containerFs.containerRootOnHost().getFileName() + ":" + pathInContainer(); - } - - private static ContainerPath resolve(ContainerFileSystem containerFs, String[] currentParts, Path other, UnixUser user) { - List<String> parts = other.isAbsolute() ? new ArrayList<>() : new ArrayList<>(Arrays.asList(currentParts)); - for (int i = 0; i < other.getNameCount(); i++) { - String part = other.getName(i).toString(); - if (part.isEmpty() || part.equals(".")) continue; - if (part.equals("..")) { - if (!parts.isEmpty()) parts.remove(parts.size() - 1); - continue; - } - parts.add(part); - } - - return new ContainerPath(containerFs, - containerFs.containerRootOnHost().resolve(String.join("/", parts)), - parts.toArray(String[]::new), - user); - } - - public static ContainerPath fromPathInContainer(ContainerFileSystem containerFs, Path pathInContainer, UnixUser user) { - if (!pathInContainer.isAbsolute()) - throw new IllegalArgumentException("Path in container must be absolute: " + pathInContainer); - return resolve(containerFs, new String[0], pathInContainer, user); - } - - public static ContainerPath fromPathOnHost(ContainerFileSystem containerFs, Path pathOnHost, UnixUser user) { - pathOnHost = pathOnHost.normalize(); - Path containerRootOnHost = containerFs.containerRootOnHost(); - Path pathUnderContainerStorage = containerRootOnHost.relativize(pathOnHost); - - if (pathUnderContainerStorage.getNameCount() == 0 || pathUnderContainerStorage.getName(0).toString().isEmpty()) - return new ContainerPath(containerFs, pathOnHost, new String[0], user); - if (pathUnderContainerStorage.getName(0).toString().equals("..")) - throw new IllegalArgumentException("Path " + pathOnHost + " is not under container root " + containerRootOnHost); - - List<String> parts = new ArrayList<>(); - for (int i = 0; i < pathUnderContainerStorage.getNameCount(); i++) - parts.add(pathUnderContainerStorage.getName(i).toString()); - return new ContainerPath(containerFs, pathOnHost, parts.toArray(String[]::new), user); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java deleted file mode 100644 index 1a9b9b60cd4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupService.java +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; - -import java.io.IOException; -import java.nio.file.attribute.GroupPrincipal; -import java.nio.file.attribute.UserPrincipal; -import java.nio.file.attribute.UserPrincipalLookupService; -import java.nio.file.attribute.UserPrincipalNotFoundException; -import java.util.Objects; -import java.util.function.Function; - -/** - * @author freva - */ -public class ContainerUserPrincipalLookupService extends UserPrincipalLookupService { - - private final UserPrincipalLookupService baseFsUserPrincipalLookupService; - private final UserScope userScope; - - ContainerUserPrincipalLookupService(UserPrincipalLookupService baseFsUserPrincipalLookupService, UserScope userScope) { - this.baseFsUserPrincipalLookupService = Objects.requireNonNull(baseFsUserPrincipalLookupService); - this.userScope = Objects.requireNonNull(userScope); - } - - public UserScope userScope() { return userScope; } - - public int userIdOnHost(int containerUid) { return userScope.namespace().userIdOnHost(containerUid); } - public int groupIdOnHost(int containerGid) { return userScope.namespace().groupIdOnHost(containerGid); } - public int userIdInContainer(int hostUid) { return userScope.namespace().userIdInContainer(hostUid); } - public int groupIdInContainer(int hostGid) { return userScope.namespace().groupIdInContainer(hostGid); } - - @Override - public ContainerUserPrincipal lookupPrincipalByName(String name) throws IOException { - int containerUid = resolveName(name, UnixUser::uid, UnixUser::name); - String user = resolveId(containerUid, UnixUser::uid, UnixUser::name); - String hostUid = String.valueOf(userIdOnHost(containerUid)); - return new ContainerUserPrincipal(containerUid, user, baseFsUserPrincipalLookupService.lookupPrincipalByName(hostUid)); - } - - @Override - public ContainerGroupPrincipal lookupPrincipalByGroupName(String group) throws IOException { - int containerGid = resolveName(group, UnixUser::gid, UnixUser::group); - String name = resolveId(containerGid, UnixUser::gid, UnixUser::group); - String hostGid = String.valueOf(groupIdOnHost(containerGid)); - return new ContainerGroupPrincipal(containerGid, name, baseFsUserPrincipalLookupService.lookupPrincipalByGroupName(hostGid)); - } - - public ContainerUserPrincipal userPrincipal(int uid, UserPrincipal baseFsPrincipal) { - String name = resolveId(uid, UnixUser::uid, UnixUser::name); - return new ContainerUserPrincipal(uid, name, baseFsPrincipal); - } - - public ContainerGroupPrincipal groupPrincipal(int gid, GroupPrincipal baseFsPrincipal) { - String name = resolveId(gid, UnixUser::gid, UnixUser::group); - return new ContainerGroupPrincipal(gid, name, baseFsPrincipal); - } - - private String resolveId(int id, Function<UnixUser, Integer> idExtractor, Function<UnixUser, String> nameExtractor) { - if (idExtractor.apply(userScope.root()) == id) return nameExtractor.apply(userScope.root()); - if (idExtractor.apply(userScope.vespa()) == id) return nameExtractor.apply(userScope.vespa()); - return String.valueOf(id); - } - - private int resolveName(String name, Function<UnixUser, Integer> idExtractor, Function<UnixUser, String> nameExtractor) throws UserPrincipalNotFoundException { - if (name.equals(nameExtractor.apply(userScope.root()))) return idExtractor.apply(userScope.root()); - if (name.equals(nameExtractor.apply(userScope.vespa()))) return idExtractor.apply(userScope.vespa()); - - try { - return Integer.parseInt(name); - } catch (NumberFormatException ignored) { - throw new UserPrincipalNotFoundException(name); - } - } - - private abstract static class NamedPrincipal implements UserPrincipal { - private final int id; - private final String name; - private final UserPrincipal baseFsPrincipal; - - private NamedPrincipal(int id, String name, UserPrincipal baseFsPrincipal) { - this.id = id; - this.name = Objects.requireNonNull(name); - this.baseFsPrincipal = Objects.requireNonNull(baseFsPrincipal); - } - - @Override - public final String getName() { - return name; - } - - public int id() { - return id; - } - - public UserPrincipal baseFsPrincipal() { - return baseFsPrincipal; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NamedPrincipal that = (NamedPrincipal) o; - return id == that.id && baseFsPrincipal.equals(that.baseFsPrincipal); - } - - @Override - public int hashCode() { - return Objects.hash(id, baseFsPrincipal); - } - - @Override - public String toString() { - return "{id=" + id + ", baseFsPrincipal=" + baseFsPrincipal + '}'; - } - } - - static final class ContainerUserPrincipal extends NamedPrincipal { - private ContainerUserPrincipal(int id, String name, UserPrincipal baseFsPrincipal) { super(id, name, baseFsPrincipal); } - } - - static final class ContainerGroupPrincipal extends NamedPrincipal implements GroupPrincipal { - private ContainerGroupPrincipal(int id, String name, GroupPrincipal baseFsPrincipal) { super(id, name, baseFsPrincipal); } - - @Override public GroupPrincipal baseFsPrincipal() { return (GroupPrincipal) super.baseFsPrincipal(); } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/package-info.java deleted file mode 100644 index 6891089ff71..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java deleted file mode 100644 index 965cd9942d6..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import ai.vespa.net.CidrBlock; -import com.google.common.net.InetAddresses; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; - -import java.io.UncheckedIOException; -import java.net.Inet4Address; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * IP addresses - IP utilities to retrieve and manipulate addresses for docker host and docker containers in a - * multi-home environment. - * <p> - * The assumption is that DNS is the source of truth for which address are assigned to the host and which - * that belongs to the containers. Only one address should be assigned to each. - * <p> - * The behavior with respect to site-local addresses are distinct for IPv4 and IPv6. For IPv4 we choose - * the site-local address (assume the public is a NAT address not assigned to the host interface (the typical aws setup)). - * <p> - * For IPv6 we disregard any site-local addresses (these are normally not in DNS anyway). - * <p> - * This class also provides some utilities for prefix translation. - * - * @author smorgrav - */ -public interface IPAddresses { - CidrBlock gcpInternalBlock = CidrBlock.fromString("2600:2d00::/32"); - - InetAddress[] getAddresses(String hostname); - - default Optional<InetAddress> getAddress(String hostname, IPVersion ipVersion) { - return ipVersion == IPVersion.IPv6 - ? getIPv6Address(hostname).map(InetAddress.class::cast) - : getIPv4Address(hostname).map(InetAddress.class::cast); - } - - /** - * Returns a list of string representation of the IP addresses (RFC 5952 compact format) - */ - default List<String> getAddresses(String hostname, IPVersion ipVersion) { - return Stream.of(getAddresses(hostname)) - .filter(inetAddress -> isOfType(inetAddress, ipVersion)) - .map(InetAddresses::toAddrString) - .toList(); - } - - /** - * Get the IPv6 address for the host if any. - * - * @throws ConvergenceException if multiple addresses are found - */ - default Optional<Inet6Address> getIPv6Address(String hostname) { - List<Inet6Address> ipv6addresses = Stream.of(getAddresses(hostname)) - .filter(Inet6Address.class::isInstance) - .filter(inetAddress -> !inetAddress.isLoopbackAddress()) - .map(Inet6Address.class::cast) - .filter(inetAddress -> !inetAddress.isLinkLocalAddress()) - .filter(inetAddress -> !inetAddress.isSiteLocalAddress()) - .filter(inet6Address -> !gcpInternalBlock.contains(inet6Address)) - .toList(); - - if (ipv6addresses.size() <= 1) return ipv6addresses.stream().findFirst(); - - String addresses = ipv6addresses.stream().map(InetAddresses::toAddrString).collect(Collectors.joining(",")); - throw ConvergenceException.ofError( - String.format( - "Multiple IPv6 addresses found: %s. Perhaps a missing DNS entry or multiple AAAA records in DNS?", - addresses)); - } - - /** Returns the hostname of given inetAddress */ - default String getHostname(InetAddress inetAddress) { - String hostname = inetAddress.getHostName(); - if (hostname.equals(inetAddress.getHostAddress())) { - throw new IllegalArgumentException("Could not find hostname for address " + inetAddress.getHostAddress()); - } - return hostname; - } - - /** - * Get the IPv4 address for the host if any. - * - * @throws ConvergenceException if multiple site-local addresses are found - */ - default Optional<Inet4Address> getIPv4Address(String hostname) { - List<Inet4Address> ipv4Addresses = Stream.of(getAddresses(hostname)) - .filter(Inet4Address.class::isInstance) - .filter(inetAddress -> !inetAddress.isLoopbackAddress()) - .map(Inet4Address.class::cast) - .toList(); - - if (ipv4Addresses.size() <= 1) return ipv4Addresses.stream().findFirst(); - - List<Inet4Address> siteLocalIPv4Addresses = ipv4Addresses.stream() - .filter(InetAddress::isSiteLocalAddress) - .toList(); - - if (siteLocalIPv4Addresses.size() == 1) return Optional.of(siteLocalIPv4Addresses.get(0)); - - String addresses = ipv4Addresses.stream().map(InetAddresses::toAddrString).collect(Collectors.joining(",")); - throw ConvergenceException.ofError( - String.format( - "Multiple IPv4 addresses found: %s. Perhaps a missing DNS entry or multiple A records in DNS?", - addresses)); - } - - static boolean isOfType(InetAddress address, IPVersion ipVersion) { - if (ipVersion.equals(IPVersion.IPv4) && address instanceof Inet4Address) return true; - if (ipVersion.equals(IPVersion.IPv6) && address instanceof Inet6Address) return true; - return false; - } - - /** - * For NPTed networks we want to find the private address from a public. - * - * @param address The original address to translate - * @param prefix The prefix address - * @param subnetSizeInBytes in bits - e.g a /64 subnet equals 8 bytes - * @return The translated address - * @throws ConvergenceException if - */ - static InetAddress prefixTranslate(InetAddress address, InetAddress prefix, int subnetSizeInBytes) { - return prefixTranslate(address.getAddress(), prefix.getAddress(), subnetSizeInBytes); - } - - static InetAddress prefixTranslate(byte[] address, byte[] prefix, int nofBytes) { - System.arraycopy(prefix, 0, address, 0, nofBytes); - try { - return InetAddress.getByAddress(address); - } catch (UnknownHostException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java deleted file mode 100644 index 4680502cee7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import java.io.UncheckedIOException; -import java.net.InetAddress; -import java.net.UnknownHostException; - -/** - * @author smorgrav - */ -public class IPAddressesImpl implements IPAddresses { - - @Override - public InetAddress[] getAddresses(String hostname) { - try { - return InetAddress.getAllByName(hostname); - } catch (UnknownHostException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java deleted file mode 100644 index eb92cbdd303..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import com.google.common.net.InetAddresses; - -import java.net.Inet4Address; -import java.net.InetAddress; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Strong type IPv4 and IPv6 with common executables for ip related commands. - * - * @author smorgrav - */ -public enum IPVersion { - - IPv6(6, "ip6tables", "ip -6", "ipv6-icmp", 128, "icmp6-port-unreachable", "ip6tables-restore", "fe80::/10"), - IPv4(4, "iptables", "ip", "icmp", 32, "icmp-port-unreachable", "iptables-restore", "169.254.0.0/16"); - - private static final Pattern cidrNotationPattern = Pattern.compile("/\\d+$"); - - IPVersion(int version, String iptablesCmd, String ipCmd, - String icmpProtocol, int size, String icmpPortUnreachable, - String iptablesRestore, String linkLocalCidr) { - this.version = version; - this.ipCmd = ipCmd; - this.iptablesCmd = iptablesCmd; - this.icmpProtocol = icmpProtocol; - this.size = size; - this.icmpPortUnreachable = icmpPortUnreachable; - this.iptablesRestore = iptablesRestore; - this.linkLocalCidr = linkLocalCidr; - } - - private final int version; - private final String iptablesCmd; - private final String ipCmd; - private final String icmpProtocol; - private final int size; - private final String icmpPortUnreachable; - private final String iptablesRestore; - private final String linkLocalCidr; - - /** The ID of the IP version, either IPv4 or IPv6. */ - public String id() { return "IPv" + version; } - - /** The IP version, either 4 or 6 */ - public int version() { return version; } - - public String versionString() { return String.valueOf(version); } - public String iptablesCmd() { return iptablesCmd; } - public String iptablesRestore() { return iptablesRestore;} - public String ipCmd() { return ipCmd; } - public String icmpProtocol() { return icmpProtocol; } - public String singleHostCidr() { return "/" + size; } - public String icmpPortUnreachable() { return icmpPortUnreachable; } - - /** The address size (in bits) of the IP version: 32 or 128. */ - public int addressSize() { return size; } - - /** Both IPv4 and IPv6 have exactly one link-local address space: 169.254.0.0/16 or fe80::/10. */ - public String linkLocalAddressCidr() { return linkLocalCidr; } - - public boolean match(InetAddress address) { - return this == IPVersion.get(address); - } - - public boolean match(String address) { - return this == IPVersion.get(address); - } - - public static IPVersion get(String address) { - Matcher matcher = cidrNotationPattern.matcher(address); - if (matcher.find()) { - address = matcher.replaceFirst(""); - } - return get(InetAddresses.forString(address)); - } - - public static IPVersion get(InetAddress address) { - return address instanceof Inet4Address ? IPv4 : IPv6; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java deleted file mode 100644 index 1186a58f53d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import com.google.common.net.InetAddresses; - -import java.net.Inet4Address; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.util.Objects; - -/** - * Encapsulates an IP address and its version along with some convenience methods. - * Default sorting is by version (IPv6 first), then by address. - * - * @author gjoranv - */ -public class VersionedIpAddress implements Comparable<VersionedIpAddress> { - - private final InetAddress address; - private final IPVersion version; - - private VersionedIpAddress(InetAddress address) { - this.address = Objects.requireNonNull(address); - version = getVersionOrThrow(address); - } - - public static VersionedIpAddress from(InetAddress address) { - return new VersionedIpAddress(address); - } - - public static VersionedIpAddress from(String address) { - return from(InetAddresses.forString(address)); - } - - public IPVersion version() { - return version; - } - - public String asString() { - return InetAddresses.toAddrString(address); - } - - public String asEndpoint(int port) { - var format = (version == IPVersion.IPv6) ? "[%s]:%d" : "%s:%d"; - return String.format(format, asString(), port); - } - - @Override - public int compareTo(VersionedIpAddress o) { - int version = version().compareTo(o.version()); - return (version != 0) ? version : asString().compareTo(o.asString()); - } - - @Override - public String toString() { - return "VersionedIpAddress{" + - "address=" + address + - ", version=" + version + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - VersionedIpAddress that = (VersionedIpAddress) o; - return address.equals(that.address) && version == that.version; - } - - @Override - public int hashCode() { - return Objects.hash(address, version); - } - - private static IPVersion getVersionOrThrow(InetAddress address) { - if (address instanceof Inet4Address) { - return IPVersion.IPv4; - } else if (address instanceof Inet6Address) { - return IPVersion.IPv6; - } else { - throw new IllegalArgumentException("Unknown IP version for " + InetAddresses.toAddrString(address) + " of class " + address.getClass().getName()); - } - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/package-info.java deleted file mode 100644 index 9533b7240c4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/package-info.java deleted file mode 100644 index 572182f7991..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author bjorncs - */ -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java deleted file mode 100644 index 007547aa41b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -/** - * @author hakonhall - */ -public interface ChildProcess2 extends AutoCloseable { - void waitForTermination(); - int exitCode(); - String getOutput(); - - /** Close/cleanup any resources held. Must not throw an exception. */ - @Override - void close(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java deleted file mode 100644 index 8574028b6d7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.jdisc.Timer; -import java.util.logging.Level; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author hakonhall - */ -public class ChildProcess2Impl implements ChildProcess2 { - private static final Logger logger = Logger.getLogger(ChildProcess2Impl.class.getName()); - - private final CommandLine commandLine; - private final ProcessApi2 process; - private final Path outputPath; - private final Timer timer; - - public ChildProcess2Impl(CommandLine commandLine, - ProcessApi2 process, - Path outputPath, - Timer timer) { - this.commandLine = commandLine; - this.process = process; - this.outputPath = outputPath; - this.timer = timer; - } - - @Override - public void waitForTermination() { - Duration timeoutDuration = commandLine.getTimeout(); - Instant timeout = timer.currentTime().plus(timeoutDuration); - long maxOutputBytes = commandLine.getMaxOutputBytes(); - - // How frequently do we want to wake up and check the output file size? - final Duration pollInterval = Duration.ofSeconds(10); - - boolean hasTerminated = false; - while (!hasTerminated) { - Instant now = timer.currentTime(); - long sleepPeriodMillis = pollInterval.toMillis(); - if (now.plusMillis(sleepPeriodMillis).isAfter(timeout)) { - sleepPeriodMillis = Duration.between(now, timeout).toMillis(); - - if (sleepPeriodMillis <= 0) { - gracefullyKill(); - throw new TimeoutChildProcessException( - timeoutDuration, commandLine.toString(), getOutput()); - } - } - - try { - hasTerminated = process.waitFor(sleepPeriodMillis, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - // Ignore, just loop around. - continue; - } - - // Always check output file size to ensure we don't load too much into memory. - long sizeInBytes = uncheck(() -> Files.size(outputPath)); - if (sizeInBytes > maxOutputBytes) { - gracefullyKill(); - throw new LargeOutputChildProcessException( - sizeInBytes, commandLine.toString(), getOutput()); - } - } - } - - @Override - public int exitCode() { - return process.exitValue(); - } - - @Override - public String getOutput() { - byte[] bytes = uncheck(() -> Files.readAllBytes(outputPath)); - return new String(bytes, commandLine.getOutputEncoding()); - } - - @Override - public void close() { - try { - if (commandLine.getOutputFile().isEmpty()) - Files.delete(outputPath); - } catch (Throwable t) { - logger.log(Level.WARNING, "Failed to delete " + outputPath, t); - } - } - - Path getOutputPath() { - return outputPath; - } - - private void gracefullyKill() { - process.destroy(); - - Duration maxWaitAfterSigTerm = commandLine.getSigTermGracePeriod(); - Instant timeout = timer.currentTime().plus(maxWaitAfterSigTerm); - if (!waitForTermination(timeout)) { - process.destroyForcibly(); - - // If waiting for the process now takes a long time, it's probably a kernel issue - // or huge core is getting dumped. - Duration maxWaitAfterSigKill = commandLine.getSigKillGracePeriod(); - if (!waitForTermination(timer.currentTime().plus(maxWaitAfterSigKill))) { - throw new UnkillableChildProcessException( - maxWaitAfterSigTerm, - maxWaitAfterSigKill, - commandLine.toString(), - getOutput()); - } - } - } - - /** @return true if process terminated, false on timeout. */ - private boolean waitForTermination(Instant timeout) { - while (true) { - long waitDurationMillis = Duration.between(timer.currentTime(), timeout).toMillis(); - if (waitDurationMillis <= 0) { - return false; - } - - try { - return process.waitFor(waitDurationMillis, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - // ignore - } - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java deleted file mode 100644 index 9a0c08a8596..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.text.internal.SnippetGenerator; - -/** - * Base class for child process related exceptions, with a util to build an error message - * that includes a large part of the output. - * - * @author hakonhall - */ -@SuppressWarnings("serial") -public abstract class ChildProcessException extends RuntimeException { - private static final SnippetGenerator snippetGenerator = new SnippetGenerator(); - - /** - * An exception with a message of the following format: - * Command 'COMMANDLINE' PROBLEM: stdout/stderr: 'OUTPUT' - * - * If the output of the terminated command is too large it will be sampled. - * - * @param problem E.g. "terminated with exit code 1" - * @param commandLine The command that failed in a concise (e.g. shell-like) format - * @param possiblyHugeOutput The output of the command - */ - protected ChildProcessException(String problem, String commandLine, String possiblyHugeOutput) { - super(makeSnippet(problem, commandLine, possiblyHugeOutput)); - } - - protected ChildProcessException(RuntimeException cause, - String problem, - String commandLine, - String possiblyHugeOutput) { - super(makeSnippet(problem, commandLine, possiblyHugeOutput), cause); - } - - private static String makeSnippet(String problem, String commandLine, String possiblyHugeOutput) { - return "Command '" + - commandLine + - "' " + - problem + - ": stdout/stderr: '" + - snippetGenerator.makeSnippet(possiblyHugeOutput, 500) + - "'"; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java deleted file mode 100644 index 2d1fe1f24bd..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -/** - * The child process terminated with a non-zero exit code. - * - * @author hakonhall - */ -@SuppressWarnings("serial") -public class ChildProcessFailureException extends ChildProcessException { - ChildProcessFailureException(int exitCode, String commandLine, String possiblyHugeOutput) { - super("terminated with exit code " + exitCode, commandLine, possiblyHugeOutput); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java deleted file mode 100644 index 516b50dc601..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeMap; -import java.util.function.Predicate; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * A CommandLine is used to specify and execute a shell-like program in a child process, - * and capture its output. - * - * @author hakonhall - */ -public class CommandLine { - private static final Logger logger = Logger.getLogger(CommandLine.class.getName()); - private static final Pattern UNESCAPED_ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=!@%/+:.,_-]+$"); - - /** The default timeout. See setTimeout() for details. */ - public static final Duration DEFAULT_TIMEOUT = Duration.ofMinutes(10); - - /** The default maximum number of output bytes. See setMaxOutputBytes() for details. */ - public static final long DEFAULT_MAX_OUTPUT_BYTES = 1024 * 1024 * 1024; // 1 Gb - - /** - * The default grace period after SIGTERM has been sent during a graceful kill. - * See setSigTermGracePeriod for details. - */ - public static final Duration DEFAULT_SIGTERM_GRACE_PERIOD = Duration.ofMinutes(1); - - /** - * The default grace period after SIGKILL has been sent during a graceful kill. - * See setSigKillGracePeriod for details. - */ - public static final Duration DEFAULT_SIGKILL_GRACE_PERIOD = Duration.ofMinutes(30); - - private final List<String> arguments = new ArrayList<>(); - private final Set<Integer> censoredArgumentIndices = new HashSet<>(); - private final TreeMap<String, String> environment = new TreeMap<>(); - private final TaskContext taskContext; - private final ProcessFactory processFactory; - - private boolean redirectStderrToStdoutInsteadOfDiscard = true; - private boolean executeSilentlyCalled = false; - private Optional<Path> outputFile = Optional.empty(); - private Charset outputEncoding = StandardCharsets.UTF_8; - private Duration timeout = DEFAULT_TIMEOUT; - private long maxOutputBytes = DEFAULT_MAX_OUTPUT_BYTES; - private Duration sigTermGracePeriod = DEFAULT_SIGTERM_GRACE_PERIOD; - private Duration sigKillGracePeriod = DEFAULT_SIGKILL_GRACE_PERIOD; - private Predicate<Integer> successfulExitCodePredicate = code -> code == 0; - private boolean waitForTermination = true; - - public CommandLine(TaskContext taskContext, ProcessFactory processFactory) { - this.taskContext = taskContext; - this.processFactory = processFactory; - } - - /** Add arguments to the command. The first argument in the first call to add() is the program. */ - public CommandLine add(String... arguments) { return add(List.of(arguments)); } - - /** Add arguments to the command. The first argument in the first call to add() is the program. */ - public CommandLine add(Collection<String> arguments) { - this.arguments.addAll(arguments); - return this; - } - - /** Add arguments by splitting arguments by space. */ - public CommandLine addTokens(String arguments) { - return add(arguments.split("\\s+")); - } - - /** Set an environment variable, overriding any existing. */ - public CommandLine setEnvironmentVariable(String name, String value) { - if (name.indexOf('=') != -1) { - throw new IllegalArgumentException("name contains '=': " + name); - } - Objects.requireNonNull(value, "cannot set environment variable to null"); - - environment.put(name, value); - return this; - } - - public CommandLine removeEnvironmentVariable(String name) { - if (name.indexOf('=') != -1) { - throw new IllegalArgumentException("name contains '=': " + name); - } - environment.put(name, null); - return this; - } - - /** Censor (prevent logging of) the last argument added to this */ - public CommandLine censorArgument() { - censoredArgumentIndices.add(arguments.size() - 1); - return this; - } - - /** - * Execute a shell-like program in a child process: - * - the program is recorded and logged as modifying the system, but see executeSilently(). - * - the program's stderr is redirected to stdout, but see discardStderr(). - * - the program's output is assumed to be UTF-8, but see setOutputEncoding(). - * - the program must terminate with exit code 0, but see ignoreExitCode(). - * - the output of the program will be accessible in the returned CommandResult. - * - * Footnote 1: As a safety measure the size of the output is capped, and the program is - * only allowed to execute up to a timeout. The defaults are set high so you typically do - * not have to worry about reaching these limits, but otherwise see setMaxOutputBytes() - * and setTimeout(), respectively. - * - * Footnote 2: If the child process is forced to be killed due to footnote 1, then - * setSigTermGracePeriod() and setSigKillGracePeriod() can be used to tweak how much time - * is given to the program to shut down. Again, the defaults should be reasonable. - */ - public CommandResult execute() { - taskContext.recordSystemModification(logger, "Executing command: " + toString()); - return doExecute(); - } - - /** - * Same as execute(), except it will not record the program as modifying the system. - * - * If the program is later found to have modified the system, or otherwise worthy of - * a record, call recordSilentExecutionAsSystemModification(). - */ - public CommandResult executeSilently() { - executeSilentlyCalled = true; - return doExecute(); - } - - public static class Options { - private boolean silent = false; - - public Options() {} - - /** Invoke {@link #executeSilently()} instead of {@link #execute()} (default). */ - public Options setSilent(boolean silent) { - this.silent = silent; - return this; - } - } - - /** Convenience method to bundle up a bunch of calls on this into an options object. */ - public CommandResult execute(Options options) { - return options.silent ? executeSilently() : execute(); - } - - /** - * Record an already executed executeSilently() as having modified the system. - * For instance with YUM it is not known until after a 'yum install' whether it - * modified the system. - */ - public void recordSilentExecutionAsSystemModification() { - if (!executeSilentlyCalled) { - throw new IllegalStateException("executeSilently has not been called"); - } - // Disallow multiple consecutive calls to this method without an intervening call - // to executeSilently(). - executeSilentlyCalled = false; - - taskContext.recordSystemModification(logger, "Executed command: " + toString()); - } - - /** - * The first argument of the command specifies the program and is either the program's - * filename (in case the environment variable PATH will be used to search for the program - * file) or a path with the last component being the program's filename. - * - * @return The filename of the program. - */ - public String programName() { - if (arguments.isEmpty()) { - throw new IllegalStateException( - "The program name cannot be determined yet as no arguments have been given"); - } - String path = arguments.get(0); - int lastIndex = path.lastIndexOf('/'); - if (lastIndex == -1) { - return path; - } else { - return path.substring(lastIndex + 1); - } - } - - /** Returns a shell-like representation of the command. */ - @Override - public String toString() { - return toString(true); - } - - String toString(boolean censor) { - var command = new StringBuilder(); - - if (!environment.isEmpty()) { - // Pretend environment is propagated through the env program for display purposes - command.append(environment.entrySet().stream() - .map(entry -> { - if (entry.getValue() == null) { - return "-u " + maybeEscapeArgument(entry.getKey()); - } else { - return maybeEscapeArgument(entry.getKey() + "=" + entry.getValue()); - } - }) - .collect(Collectors.joining(" ", "env ", " "))); - } - - for (int i = 0; i < arguments.size(); i++) { - if (censor && censoredArgumentIndices.contains(i)) { - command.append("<censored>"); - } else { - command.append(maybeEscapeArgument(arguments.get(i))); - } - if (i < arguments.size() - 1) { - command.append(" "); - } - } - - // Note: Both of these cannot be confused with an argument since they would - // require escaping. - command.append(redirectStderrToStdoutInsteadOfDiscard ? " 2>&1" : " 2>/dev/null"); - - return command.toString(); - } - - - /** - * By default, stderr is redirected to stderr. This method will instead discard stderr. - */ - public CommandLine discardStderr() { - this.redirectStderrToStdoutInsteadOfDiscard = false; - return this; - } - - /** - * By default, a non-zero exit code will cause the command execution to fail. This method - * will instead ignore the exit code. - */ - public CommandLine ignoreExitCode() { - this.successfulExitCodePredicate = code -> true; - return this; - } - - /** - * By default, a non-zero exit code causes the command execution to fail. This method - * will override that predicate. - */ - public CommandLine setSuccessfulExitCodePredicate(Predicate<Integer> successPredicate) { - successfulExitCodePredicate = successPredicate; - return this; - } - - /** - * By default, the output of the command is parsed as UTF-8. This method will set a - * different encoding. - */ - public CommandLine setOutputEncoding(Charset outputEncoding) { - this.outputEncoding = outputEncoding; - return this; - } - - /** - * By default, the output of the command is piped to a temporary file, which is deleted - * when execution ends. This method will cause output to be piped to the given path - * instead, and the file will not be removed. - */ - public CommandLine setOutputFile(Path outputFile) { - this.outputFile = Optional.of(outputFile); - return this; - } - - /** - * By default, the command will be gracefully killed after DEFAULT_TIMEOUT. This method - * overrides that default. - */ - public CommandLine setTimeout(Duration timeout) { - this.timeout = timeout; - return this; - } - - /** - * By default, the command will be gracefully killed if it ever outputs more than - * DEFAULT_MAX_OUTPUT_BYTES. This method overrides that default. - */ - public CommandLine setMaxOutputBytes(long maxOutputBytes) { - this.maxOutputBytes = maxOutputBytes; - return this; - } - - /** - * By default, if the program needs to be gracefully killed it will wait up to - * DEFAULT_SIGTERM_GRACE_PERIOD for the program to exit after it has been killed with - * the SIGTERM signal. - */ - public CommandLine setSigTermGracePeriod(Duration period) { - this.sigTermGracePeriod = period; - return this; - } - - public CommandLine setSigKillGracePeriod(Duration period) { - this.sigKillGracePeriod = period; - return this; - } - - /** - * WARNING: This will leave the child as a zombie process until this process dies. - * I.e. only use this just before or a limited number of times per host admin restart. - */ - public CommandLine doNotWaitForTermination() { - this.waitForTermination = false; - return this; - } - - public List<String> getArguments() { return Collections.unmodifiableList(arguments); } - - /** Returns a copy of the environment overrides. A null value means the environment variable should be removed. */ - public TreeMap<String, String> getEnvironmentOverrides() { return new TreeMap<>(environment); } - - // Accessor fields necessary for classes in this package. Could be public if necessary. - boolean getRedirectStderrToStdoutInsteadOfDiscard() { return redirectStderrToStdoutInsteadOfDiscard; } - Predicate<Integer> getSuccessfulExitCodePredicate() { return successfulExitCodePredicate; } - Optional<Path> getOutputFile() { return outputFile; } - Charset getOutputEncoding() { return outputEncoding; } - Duration getTimeout() { return timeout; } - long getMaxOutputBytes() { return maxOutputBytes; } - Duration getSigTermGracePeriod() { return sigTermGracePeriod; } - Duration getSigKillGracePeriod() { return sigKillGracePeriod; } - - private CommandResult doExecute() { - try (ChildProcess2 child = processFactory.spawn(this)) { - if (!waitForTermination) { - return new CommandResult(this, 0, ""); - } - - child.waitForTermination(); - int exitCode = child.exitCode(); - if (!successfulExitCodePredicate.test(exitCode)) { - throw new ChildProcessFailureException(exitCode, toString(), child.getOutput()); - } - - String output = child.getOutput(); - return new CommandResult(this, exitCode, output); - } - } - - private static String maybeEscapeArgument(String argument) { - if (UNESCAPED_ARGUMENT_PATTERN.matcher(argument).matches()) { - return argument; - } else { - return escapeArgument(argument); - } - } - - private static String escapeArgument(String argument) { - StringBuilder doubleQuoteEscaped = new StringBuilder(argument.length() + 10); - - for (int i = 0; i < argument.length(); ++i) { - char c = argument.charAt(i); - switch (c) { - case '"', '\\' -> doubleQuoteEscaped.append("\\").append(c); - default -> doubleQuoteEscaped.append(c); - } - } - - return "\"" + doubleQuoteEscaped + "\""; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java deleted file mode 100644 index c4f3229792b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import java.util.List; -import java.util.function.Function; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * A CommandResult is the result of the execution of a CommandLine. - * - * @author hakonhall - */ -public class CommandResult { - private static final Pattern NEWLINE = Pattern.compile("\\n"); - - private final CommandLine commandLine; - private final int exitCode; - private final String output; - - public CommandResult(CommandLine commandLine, int exitCode, String output) { - this.commandLine = commandLine; - this.exitCode = exitCode; - this.output = output; - } - - public int getExitCode() { - return exitCode; - } - - /** Returns the output with leading and trailing white-space removed. */ - public String getOutput() { return output.trim(); } - - public String getUntrimmedOutput() { return output; } - - /** Returns the output lines of the command, omitting trailing empty lines. */ - public List<String> getOutputLines() { - return getOutputLinesStream().toList(); - } - - /** Returns the output lines as a stream, omitting trailing empty lines. */ - public Stream<String> getOutputLinesStream() { - if (output.isEmpty()) { - // For some reason an empty string => one-element list. - return Stream.empty(); - } - - // For some reason this removes trailing empty elements, but that's OK with us. - return NEWLINE.splitAsStream(output); - } - - /** - * Map this CommandResult to an instance of type R. - * - * If a RuntimeException is thrown by the mapper, it is wrapped in an - * UnexpectedOutputException that includes a snippet of the output in the message. - * - * This method is intended to be used as part of the verification of the output. - */ - public <R> R map(Function<CommandResult, R> mapper) { - try { - return mapper.apply(this); - } catch (RuntimeException e) { - throw new UnexpectedOutputException(e, "Failed to map output", commandLine.toString(), output); - } - } - - /** - * Map the output to an instance of type R according to mapper, wrapping any - * RuntimeException in UnexpectedOutputException w/output snippet. See map() for details. - */ - public <R> R mapOutput(Function<String, R> mapper) { return map(result -> mapper.apply(result.getOutput())); } - - /** - * Map each output line to an instance of type R according to mapper, wrapping any - * RuntimeException in UnexpectedOutputException w/output snippet. See map() for details. - */ - public <R> List<R> mapEachLine(Function<String, R> mapper) { - return map(result -> result.getOutputLinesStream().map(mapper).toList()); - } - - /** - * Convenience method for getting the CommandLine, whose execution resulted in - * this CommandResult instance. - * - * Warning: the CommandLine is mutable and may be changed by the caller of the execution - * through other references! This is just a convenience method for getting that instance. - */ - public CommandLine getCommandLine() { return commandLine; } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java deleted file mode 100644 index 440928b5762..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -/** - * Exception thrown if the output of the child process is larger than the maximum limit. - * - * @author hakonhall - */ -@SuppressWarnings("serial") -public class LargeOutputChildProcessException extends ChildProcessException { - LargeOutputChildProcessException(long maxFileSize, String commandLine, String possiblyHugeOutput) { - super("output more than " + maxFileSize + " bytes", commandLine, possiblyHugeOutput); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java deleted file mode 100644 index 006f1373e0f..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import java.util.concurrent.TimeUnit; - -/** - * Process abstraction. - * - * @author hakonhall - */ -public interface ProcessApi2 { - boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException; - int exitValue(); - void destroy(); - void destroyForcibly(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java deleted file mode 100644 index 0e4bc799007..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import java.util.concurrent.TimeUnit; - -/** - * @author hakonhall - */ -public class ProcessApi2Impl implements ProcessApi2 { - private final Process process; - - ProcessApi2Impl(Process process) { - this.process = process; - } - - @Override - public boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException { - return process.waitFor(timeout, unit); - } - - @Override - public int exitValue() { - return process.exitValue(); - } - - @Override - public void destroy() { - process.destroy(); - } - - @Override - public void destroyForcibly() { - process.destroyForcibly(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java deleted file mode 100644 index c09d6b543c3..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -/** - * @author hakonhall - */ -public interface ProcessFactory { - ChildProcess2 spawn(CommandLine commandLine); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java deleted file mode 100644 index f4bef260ec0..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.jdisc.Timer; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author hakonhall - */ -public class ProcessFactoryImpl implements ProcessFactory { - private static final Logger logger = Logger.getLogger(ProcessFactoryImpl.class.getName()); - private static final File DEV_NULL = new File("/dev/null"); - - private final ProcessStarter processStarter; - private final Timer timer; - - ProcessFactoryImpl(ProcessStarter processStarter, Timer timer) { - this.processStarter = processStarter; - this.timer = timer; - } - - @Override - public ChildProcess2Impl spawn(CommandLine commandLine) { - List<String> arguments = commandLine.getArguments(); - if (arguments.isEmpty()) { - throw new IllegalArgumentException("No arguments specified - missing program to spawn"); - } - - ProcessBuilder processBuilder = new ProcessBuilder(arguments); - - for (var entry : commandLine.getEnvironmentOverrides().entrySet()) { - if (entry.getValue() == null) { - processBuilder.environment().remove(entry.getKey()); - } else { - processBuilder.environment().put(entry.getKey(), entry.getValue()); - } - } - - if (commandLine.getRedirectStderrToStdoutInsteadOfDiscard()) { - processBuilder.redirectErrorStream(true); - } else { - processBuilder.redirectError(ProcessBuilder.Redirect.to(DEV_NULL)); - } - - // The output is redirected to a file (temporary or user-defined) because: - // - We could read continuously from process.getInputStream, but that may block - // indefinitely with a faulty program. - // - If we don't read continuously from process.getInputStream, then because - // the underlying channel may be a pipe, the child may be stopped because the pipe - // is full. - // - To honor the timeout, no API can be used that may end up blocking indefinitely. - // - // Therefore, we redirect the output to a file and use waitFor w/timeout. This also - // has the benefit of allowing for inspection of the file during execution, and - // allowing the inspection of the file if it e.g. gets too large to hold in-memory. - - FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute( - PosixFilePermissions.fromString("rw-------")); - - Path outputFile = commandLine.getOutputFile() - .map(file -> { - uncheck(() -> Files.deleteIfExists(file)); - uncheck(() -> Files.createFile(file, fileAttribute)); - return file; - }) - .orElseGet(() -> { - String temporaryFilePrefix = - ProcessFactoryImpl.class.getSimpleName() + "-" + commandLine.programName() + "-"; - - return uncheck(() -> Files.createTempFile( - temporaryFilePrefix, - ".out", - fileAttribute)); - }); - - try { - processBuilder.redirectOutput(outputFile.toFile()); - ProcessApi2 process = processStarter.start(processBuilder); - return new ChildProcess2Impl(commandLine, process, outputFile, timer); - } catch (RuntimeException | Error throwable) { - try { - if (commandLine.getOutputFile().isEmpty()) - Files.delete(outputFile); - } catch (IOException ioException) { - logger.log(Level.WARNING, "Failed to delete temporary file at " + - outputFile, ioException); - } - throw throwable; - } - - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java deleted file mode 100644 index fc78b5d3e72..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -/** - * @author hakonhall - */ -public interface ProcessStarter { - ProcessApi2 start(ProcessBuilder processBuilder); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java deleted file mode 100644 index 644e5876eb7..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import java.util.logging.Level; - -import java.util.logging.Logger; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author hakonhall - */ -public class ProcessStarterImpl implements ProcessStarter { - private static final Logger logger = Logger.getLogger(ProcessStarterImpl.class.getName()); - - @Override - public ProcessApi2 start(ProcessBuilder processBuilder) { - if (logger.isLoggable(Level.FINE)) { - logger.log(Level.FINE, "Spawning process: " + processBuilder.command()); - } - - Process process = uncheck(processBuilder::start); - return new ProcessApi2Impl(process); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java deleted file mode 100644 index 1cf6b533d5a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -/** - * A Terminal is a light-weight terminal-like interface for executing shell-like programs. - * - * @author hakonhall - */ -public interface Terminal { - CommandLine newCommandLine(TaskContext taskContext); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java deleted file mode 100644 index e13e30d9c75..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -/** - * @author hakonhall - */ -public class TerminalImpl implements Terminal { - private final ProcessFactory processFactory; - - public TerminalImpl(Timer timer) { - this(new ProcessFactoryImpl(new ProcessStarterImpl(), timer)); - } - - /** For testing. */ - public TerminalImpl(ProcessFactory processFactory) { - this.processFactory = processFactory; - } - - @Override - public CommandLine newCommandLine(TaskContext taskContext) { - return new CommandLine(taskContext, processFactory); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java deleted file mode 100644 index 8490bc01f56..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import java.util.Optional; - -/** - * @author hakonhall - */ -public class TestChildProcess2 implements ChildProcess2 { - private final int exitCode; - private final String output; - private Optional<RuntimeException> exceptionToThrowInWaitForTermination = Optional.empty(); - private boolean closeCalled = false; - - public TestChildProcess2(int exitCode, String output) { - this.exitCode = exitCode; - this.output = output; - } - - public void throwInWaitForTermination(RuntimeException e) { - this.exceptionToThrowInWaitForTermination = Optional.of(e); - } - - @Override - public void waitForTermination() { - if (exceptionToThrowInWaitForTermination.isPresent()) { - throw exceptionToThrowInWaitForTermination.get(); - } - } - - @Override - public int exitCode() { - return exitCode; - } - - @Override - public String getOutput() { - return output; - } - - @Override - public void close() { - if (closeCalled) { - throw new IllegalStateException("close already called"); - } - closeCalled = true; - } - - public boolean closeCalled() { - return closeCalled; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java deleted file mode 100644 index 4e831dc2865..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; - -/** - * @author hakonhall - */ -public class TestProcessFactory implements ProcessFactory { - private static class SpawnCall { - private final String commandDescription; - private final Function<CommandLine, ChildProcess2> callback; - - private SpawnCall(String commandDescription, - Function<CommandLine, ChildProcess2> callback) { - this.commandDescription = commandDescription; - this.callback = callback; - } - } - private final List<SpawnCall> expectedSpawnCalls = new ArrayList<>(); - private final List<CommandLine> spawnCommandLines = new ArrayList<>(); - - private boolean muteVerifyAllCommandsExecuted = false; - - /** Forward call to spawn() to callback. */ - public TestProcessFactory interceptSpawn(String commandDescription, - Function<CommandLine, ChildProcess2> callback) { - expectedSpawnCalls.add(new SpawnCall(commandDescription, callback)); - return this; - } - - // Convenience method for the caller to avoid having to create a TestChildProcess2 instance. - public TestProcessFactory expectSpawn(String commandLineString, TestChildProcess2 toReturn) { - int commandIndex = expectedSpawnCalls.size(); - return interceptSpawn( - commandLineString, - commandLine -> defaultSpawn(commandLine, commandLineString, toReturn, commandIndex)); - } - - // Convenience method for the caller to avoid having to create a TestChildProcess2 instance. - public TestProcessFactory expectSpawn(String commandLine, int exitCode, String output) { - return expectSpawn(commandLine, new TestChildProcess2(exitCode, output)); - } - - /** Ignore the CommandLine passed to spawn(), just return successfully with the given output. */ - public TestProcessFactory ignoreSpawn(String output) { - return interceptSpawn( - "[call index " + expectedSpawnCalls.size() + "]", - commandLine -> new TestChildProcess2(0, output)); - } - - public TestProcessFactory ignoreSpawn() { - return ignoreSpawn(""); - } - - public void verifyAllCommandsExecuted() { - if (muteVerifyAllCommandsExecuted) return; - - if (spawnCommandLines.size() < expectedSpawnCalls.size()) { - int missingCommandIndex = spawnCommandLines.size(); - throw new IllegalStateException("Command #" + missingCommandIndex + - " never executed: " + - expectedSpawnCalls.get(missingCommandIndex).commandDescription + - "\nExpected commands:\n" + getExpectedCommandLines() + - "\nActual commands:\n" + spawnCommandLines); - } - } - - /** - * WARNING: CommandLine is mutable, and e.g. reusing a CommandLine for the next call - * would make the CommandLine in this list no longer reflect the original CommandLine. - */ - public List<CommandLine> getMutableCommandLines() { - return spawnCommandLines; - } - - @Override - public ChildProcess2 spawn(CommandLine commandLine) { - String commandLineString = commandLine.toString(false); - if (spawnCommandLines.size() + 1 > expectedSpawnCalls.size()) { - throw new IllegalStateException("Too many invocations: " + commandLineString); - } - spawnCommandLines.add(commandLine); - - return expectedSpawnCalls.get(spawnCommandLines.size() - 1).callback.apply(commandLine); - } - - private ChildProcess2 defaultSpawn(CommandLine commandLine, - String expectedCommandLineString, - ChildProcess2 toReturn, - int commandSequenceNumber) { - String actualCommandLineString = commandLine.toString(false); - if (!Objects.equals(actualCommandLineString, expectedCommandLineString)) { - muteVerifyAllCommandsExecuted = true; - throw new IllegalArgumentException("Expected command #" + commandSequenceNumber + " to be: \n" + - " \"" + expectedCommandLineString + "\"\n" + - "but got:\n" + - " \"" + actualCommandLineString + "\""); - } - - return toReturn; - } - - private List<String> getExpectedCommandLines() { - return expectedSpawnCalls.stream() - .map(spawnCall -> spawnCall.commandDescription) - .toList(); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java deleted file mode 100644 index bf231b7c35b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; - -import java.util.function.Function; - -/** - * @author hakonhall - */ -public class TestTerminal implements Terminal { - private final TerminalImpl realTerminal; - private final TestProcessFactory testProcessFactory = new TestProcessFactory(); - - public TestTerminal() { - this.realTerminal = new TerminalImpl(testProcessFactory); - } - - /** Get the TestProcessFactory the terminal was started with. */ - public TestProcessFactory getTestProcessFactory() { return testProcessFactory; } - - /** Forward call to spawn() to callback. */ - public TestTerminal interceptCommand(String commandDescription, - Function<CommandLine, ChildProcess2> callback) { - testProcessFactory.interceptSpawn(commandDescription, callback); - return this; - } - - /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */ - public TestTerminal expectCommand(String commandLine, TestChildProcess2 toReturn) { - testProcessFactory.expectSpawn(commandLine, toReturn); - return this; - } - - /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */ - public TestTerminal expectCommand(String commandLine, int exitCode, String output) { - testProcessFactory.expectSpawn(commandLine, new TestChildProcess2(exitCode, output)); - return this; - } - - /** Verifies command line matches commandLine, and returns successfully with output "". */ - public TestTerminal expectCommand(String commandLine) { - expectCommand(commandLine, 0, ""); - return this; - } - - /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */ - public TestTerminal ignoreCommand(String output) { - testProcessFactory.ignoreSpawn(output); - return this; - } - - /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */ - public TestTerminal ignoreCommand() { - testProcessFactory.ignoreSpawn(); - return this; - } - - public void verifyAllCommandsExecuted() { - testProcessFactory.verifyAllCommandsExecuted(); - } - - @Override - public CommandLine newCommandLine(TaskContext taskContext) { - return realTerminal.newCommandLine(taskContext); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java deleted file mode 100644 index c4c59073de8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import java.time.Duration; - -/** - * Exception thrown when a child process has taken too long to terminate, in case it has been - * forcibly killed. - * - * @author hakonhall - */ -@SuppressWarnings("serial") -public class TimeoutChildProcessException extends ChildProcessException { - TimeoutChildProcessException(Duration timeout, String commandLine, String possiblyHugeOutput) { - super("timed out after " + timeout, commandLine, possiblyHugeOutput); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException.java deleted file mode 100644 index 1829df96601..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -/** - * @author hakonhall - */ -@SuppressWarnings("serial") -public class UnexpectedOutputException extends ChildProcessException { - /** - * @param problem Problem description, e.g. "Output is not of the form ^NAME=VALUE$" - */ - public UnexpectedOutputException(String problem, String commandLine, String possiblyHugeOutput) { - super("output was not of the expected format: " + problem, commandLine, possiblyHugeOutput); - } - - /** - * @param problem Problem description, e.g. "Output is not of the form ^NAME=VALUE$" - */ - public UnexpectedOutputException(RuntimeException cause, - String problem, - String commandLine, - String possiblyHugeOutput) { - super(cause, "output was not of the expected format: " + problem, commandLine, possiblyHugeOutput); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java deleted file mode 100644 index 1b847380b47..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import java.time.Duration; - -/** - * @author hakonhall - */ -@SuppressWarnings("serial") -public class UnkillableChildProcessException extends ChildProcessException { - public UnkillableChildProcessException(Duration waitForSigTerm, - Duration waitForSigKill, - String commandLine, - String possiblyHugeOutput) { - super("did not terminate even after SIGTERM, +" + waitForSigTerm + - ", SIGKILL, and +" + waitForSigKill, - commandLine, - possiblyHugeOutput); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java deleted file mode 100644 index d03eb80af50..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java deleted file mode 100644 index 55c7b23b1e8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.systemd; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; - -import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Control the systemd system and service manager - * - * @author hakonhall - */ -public class SystemCtl { - - // Valid systemd property names from looking at a couple of services. - private static final Pattern PROPERTY_NAME_PATTERN = Pattern.compile("^[a-zA-Z]+$"); - - // Last line of `systemctl list-unit-files <unit>` prints '0 unit files listed.' - private static final Pattern UNIT_FILES_LISTED_PATTERN = Pattern.compile("([0-9]+) unit files listed\\."); - - private static final Pattern ACTIVE_STATE_PROPERTY_PATTERN = createPropertyPattern("ActiveState"); - - private final Terminal terminal; - private boolean useSudo = false; - - private static Pattern createPropertyPattern(String propertyName) { - if (!PROPERTY_NAME_PATTERN.matcher(propertyName).matches()) { - throw new IllegalArgumentException("Property name does not match " + PROPERTY_NAME_PATTERN); - } - - // Make ^ and $ match beginning and end of lines. - String regex = String.format("(?md)^%s=(.*)$", propertyName); - - return Pattern.compile(regex); - } - - public SystemCtl(Terminal terminal) { - this.terminal = terminal; - } - - /** Call all commands through sudo */ - public SystemCtl withSudo() { - this.useSudo = true; - return this; - } - - /** Returns whether this is configured to use sudo */ - public boolean useSudo() { - return useSudo; - } - - public void daemonReload(TaskContext taskContext) { - newCommandLine(taskContext).add("systemctl", "daemon-reload") - .execute(); - } - - public SystemCtlEnable enable(String unit) { return new SystemCtlEnable(unit); } - public SystemCtlDisable disable(String unit) { return new SystemCtlDisable(unit); } - public SystemCtlStart start(String unit) { return new SystemCtlStart(unit); } - public SystemCtlStop stop(String unit) { return new SystemCtlStop(unit); } - public SystemCtlRestart restart(String unit) { return new SystemCtlRestart(unit); } - public SystemCtlReload reload(String unit) { return new SystemCtlReload(unit); } - - public boolean serviceExists(TaskContext context, String unit) { - return newCommandLine(context) - .add("systemctl", "list-unit-files", unit + ".service").executeSilently() - .mapOutput(output -> { - // Last line of the form: "1 unit files listed." - Matcher matcher = UNIT_FILES_LISTED_PATTERN.matcher(output); - if (!matcher.find()) { - throw new IllegalArgumentException(); - } - - return !matcher.group(1).equals("0"); - }); - } - - /** Returns true if the unit exists and is active (i.e. running). unit is e.g. "docker". */ - public boolean isActive(TaskContext context, String unit) { - return newCommandLine(context) - .add("systemctl", "--quiet", "is-active", unit + ".service") - .ignoreExitCode() - .executeSilently() - .map(CommandResult::getExitCode) == 0; - } - - public String getServiceProperty(TaskContext context, String unit, String property) { - return newCommandLine(context) - .add("systemctl", "show", "--property", property, "--value", unit + ".service") - .executeSilently() - .getOutput(); - } - - private CommandLine newCommandLine(TaskContext context) { - var commandLine = terminal.newCommandLine(context); - if (useSudo) { - commandLine.add("sudo"); - } - return commandLine; - } - - public class SystemCtlEnable extends SystemCtlCommand { - private SystemCtlEnable(String unit) { - super("enable", unit); - } - - protected boolean isAlreadyConverged(TaskContext context) { - return isUnitEnabled(context); - } - } - - public class SystemCtlDisable extends SystemCtlCommand { - private SystemCtlDisable(String unit) { - super("disable", unit); - } - - protected boolean isAlreadyConverged(TaskContext context) { - return !isUnitEnabled(context); - } - } - - public class SystemCtlStart extends SystemCtlCommand { - private SystemCtlStart(String unit) { - super("start", unit); - } - - protected boolean isAlreadyConverged(TaskContext context) { - String activeState = getSystemCtlProperty(context, ACTIVE_STATE_PROPERTY_PATTERN); - return Objects.equals(activeState, "active"); - } - } - - public class SystemCtlStop extends SystemCtlCommand { - private SystemCtlStop(String unit) { - super("stop", unit); - } - - protected boolean isAlreadyConverged(TaskContext context) { - String activeState = getSystemCtlProperty(context, ACTIVE_STATE_PROPERTY_PATTERN); - return Objects.equals(activeState, "inactive"); - } - } - - public class SystemCtlRestart extends SystemCtlCommand { - private SystemCtlRestart(String unit) { - super("restart", unit); - } - - protected boolean isAlreadyConverged(TaskContext context) { - return false; - } - } - - public class SystemCtlReload extends SystemCtlCommand { - private SystemCtlReload(String unit) { - super("reload", unit); - } - - protected boolean isAlreadyConverged(TaskContext context) { - return false; - } - } - - public abstract class SystemCtlCommand { - - private final String command; - private final String unit; - - private SystemCtlCommand(String command, String unit) { - this.command = command; - this.unit = unit; - } - - protected abstract boolean isAlreadyConverged(TaskContext context); - - public boolean converge(TaskContext context) { - if (isAlreadyConverged(context)) { - return false; - } - newCommandLine(context).add("systemctl", command, unit) - .execute(); - return true; - } - - /** Returns true if unit is enabled */ - boolean isUnitEnabled(TaskContext context) { - return newCommandLine(context).add("systemctl", "--quiet", "is-enabled", unit) - .ignoreExitCode() - .executeSilently() - .map(CommandResult::getExitCode) == 0; - } - - /** - * @param propertyPattern Pattern to match the output of systemctl show command with - * exactly 1 group. The matchng group must exist. - * @return The matched group from the 'systemctl show' output. - */ - String getSystemCtlProperty(TaskContext context, Pattern propertyPattern) { - return newCommandLine(context).add("systemctl", "show", unit) - .executeSilently() - .mapOutput(output -> extractProperty(output, propertyPattern)); - } - } - - - /** - * Find the systemd property value of the property (given by propertyPattern) - * matching the 'systemctl show' output (given by showProcess). - */ - private static String extractProperty(String showOutput, Pattern propertyPattern) { - Matcher matcher = propertyPattern.matcher(showOutput); - if (!matcher.find()) { - throw new IllegalArgumentException("Pattern '" + propertyPattern + - "' didn't match output"); - } else if (matcher.groupCount() != 1) { - throw new IllegalArgumentException("Property pattern must have exactly 1 group"); - } - - return matcher.group(1); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java deleted file mode 100644 index 32da4f455c1..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.systemd; - -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; - -import java.util.function.Consumer; - -/** - * A {@link SystemCtl} tester that simplifies testing interaction with systemd units. - * - * @author mpolden - */ -public class SystemCtlTester extends SystemCtl { - - private final TestTerminal terminal; - - public SystemCtlTester(TestTerminal terminal) { - super(terminal); - this.terminal = terminal; - } - - public Expectation expectServiceExists(String unit) { - return new Expectation(wantedReturn -> - expectCommand("systemctl list-unit-files " + unit + ".service 2>&1", 0, (wantedReturn ? 1 : 0) + " unit files listed.")); - } - - public Expectation expectIsActive(String unit) { - return new Expectation(wantedReturn -> { - expectCommand("systemctl --quiet is-active " + unit + ".service 2>&1", wantedReturn ? 0 : 1, ""); - }); - } - - public Expectation expectEnable(String unit) { return forChangeEnabledState(unit, true); } - public Expectation expectDisable(String unit) { return forChangeEnabledState(unit, false); } - public Expectation expectStart(String unit) { return forChangeRunningState(unit, true); } - public Expectation expectStop(String unit) { return forChangeRunningState(unit, false); } - - public SystemCtlTester expectRestart(String unit) { - expectCommand("systemctl restart " + unit + " 2>&1", 0, ""); - return this; - } - - public SystemCtlTester expectReload(String unit) { - expectCommand("systemctl reload " + unit + " 2>&1", 0, ""); - return this; - } - - public SystemCtlTester expectDaemonReload() { - expectCommand("systemctl daemon-reload 2>&1", 0, ""); - return this; - } - - public SystemCtlTester expectGetServiceProperty(String unit, String property, String output) { - expectCommand("systemctl show --property " + property + " --value " + unit + ".service 2>&1", 0, output); - return this; - } - - private void expectCommand(String command, int exitCode, String output) { - terminal.expectCommand((useSudo() ? "sudo " : "") + command, exitCode, output); - } - - private Expectation forChangeEnabledState(String unit, boolean enable) { - return new Expectation(wantedReturn -> { - expectCommand("systemctl --quiet is-enabled " + unit + " 2>&1", enable != wantedReturn ? 0 : 1, ""); - if (wantedReturn) - expectCommand("systemctl " + (enable ? "enable" : "disable") + " " + unit + " 2>&1", 0, ""); - }); - } - - private Expectation forChangeRunningState(String unit, boolean start) { - return new Expectation(wantedReturn -> { - expectCommand("systemctl show " + unit + " 2>&1", 0, "ActiveState=" + (start != wantedReturn ? "active" : "inactive")); - if (wantedReturn) - expectCommand("systemctl " + (start ? "start" : "stop") + " " + unit + " 2>&1", 0, ""); - }); - } - - public class Expectation { - private final Consumer<Boolean> converger; - public Expectation(Consumer<Boolean> converger) { - this.converger = converger; - } - - /** Mock the return value of the converge(TaskContext) method for this operation (true iff system was modified) */ - public SystemCtlTester andReturn(boolean value) { - converger.accept(value); - return SystemCtlTester.this; - } - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/package-info.java deleted file mode 100644 index 465cec3c026..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util.systemd; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java deleted file mode 100644 index 2d907f79e2d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; - -/** - * @author hakonhall - */ -public class BadTemplateException extends TemplateException { - public BadTemplateException(Cursor location, String message) { - super(message + " at " + location.calculateLocation().lineAndColumnText()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java deleted file mode 100644 index 3ebac3322b4..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import java.util.Locale; - -/** - * Public methods common to both Template and ListElement. - * - * @author hakonhall - */ -public interface Form { - /** Set the value of a variable, e.g. %{=color}. */ - Template set(String name, String value); - - /** Set the value of a variable and/or if-condition. */ - default Template set(String name, boolean value) { return set(name, Boolean.toString(value)); } - - default Template set(String name, int value) { return set(name, Integer.toString(value)); } - default Template set(String name, long value) { return set(name, Long.toString(value)); } - - default Template set(String name, String format, Object first, Object... rest) { - var args = new Object[1 + rest.length]; - args[0] = first; - System.arraycopy(rest, 0, args, 1, rest.length); - var value = String.format(Locale.US, format, args); - - return set(name, value); - } - - /** Add an instance of a list section after any previously added (for the given name) */ - ListElement add(String name); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java deleted file mode 100644 index d00b66c9b24..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -import java.util.Optional; - -/** - * @author hakonhall - */ -class IfSection extends Section { - private final boolean negated; - private final String name; - private final Cursor nameOffset; - private final SectionList ifSections; - private final Optional<SectionList> elseSections; - - IfSection(CursorRange range, boolean negated, String name, Cursor nameOffset, - SectionList ifSections, Optional<SectionList> elseSections) { - super("if", range); - this.negated = negated; - this.name = name; - this.nameOffset = nameOffset; - this.ifSections = ifSections; - this.elseSections = elseSections; - } - - String name() { return name; } - Cursor nameOffset() { return nameOffset; } - - @Override - void appendTo(StringBuilder buffer) { - Optional<String> stringValue = template().getVariableValue(name); - if (stringValue.isEmpty()) - throw new TemplateNameNotSetException(name, nameOffset); - - final boolean value; - if (stringValue.get().equals("true")) { - value = true; - } else if (stringValue.get().equals("false")) { - value = false; - } else { - throw new NotBooleanValueTemplateException(name); - } - - boolean condition = negated ? !value : value; - if (condition) { - ifSections.sections().forEach(section -> section.appendTo(buffer)); - } else if (elseSections.isPresent()) { - elseSections.get().sections().forEach(section -> section.appendTo(buffer)); - } - } - - @Override - void appendCopyTo(SectionList sectionList) { - SectionList ifSectionCopy = new SectionList(ifSections.range().start(), sectionList.templateBuilder()); - ifSections.sections().forEach(section -> section.appendCopyTo(ifSectionCopy)); - - Optional<SectionList> elseSectionCopy = elseSections.map(elseSections2 -> { - SectionList elseSectionCopy2 = new SectionList(elseSections2.range().start(), - sectionList.templateBuilder()); - elseSections2.sections().forEach(section -> section.appendCopyTo(elseSectionCopy2)); - return elseSectionCopy2; - }); - - sectionList.appendIfSection(negated, name, nameOffset, range().end(), ifSectionCopy, elseSectionCopy); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java deleted file mode 100644 index e8b96d4a6b8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -/** - * @author hakonhall - */ -public class ListElement implements Form { - private final Template template; - - ListElement(Template template) { this.template = template; } - - @Override - public Template set(String name, String value) { return template.set(name, value); } - - @Override - public ListElement add(String name) { return new ListElement(template.addElement(name)); } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java deleted file mode 100644 index 512518c3a42..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author hakonhall - */ -class ListSection extends Section { - private final String name; - private final Cursor nameOffset; - private final Template body; - private final List<Template> elements = new ArrayList<>(); - - ListSection(CursorRange range, String name, Cursor nameOffset, Template body) { - super("list", range); - this.name = name; - this.nameOffset = new Cursor(nameOffset); - this.body = body; - } - - String name() { return name; } - Cursor nameOffset() { return new Cursor(nameOffset); } - - @Override - void setTemplate(Template template) { - super.setTemplate(template); - body.setParent(template); - } - - Template add() { - Template element = body.snapshot(); - element.setParent(template()); - elements.add(element); - return element; - } - - @Override - void appendTo(StringBuilder buffer) { - elements.forEach(template -> template.appendTo(buffer)); - } - - @Override - void appendCopyTo(SectionList sectionList) { - // Optimization: Reuse body in copy, since it is only used for copying. - - ListSection newSection = sectionList.appendListSection(name, nameOffset, range().end(), body); - - elements.stream() - .map(template -> { - Template templateCopy = template.snapshot(); - templateCopy.setParent(template()); - return templateCopy; - }) - .forEach(newSection.elements::add); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java deleted file mode 100644 index 50c07fd1e7e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -/** - * Represents a template literal section - * - * @see Template - * @author hakonhall - */ -class LiteralSection extends Section { - LiteralSection(CursorRange range) { - super("literal", range); - } - - @Override - void appendTo(StringBuilder buffer) { - range().appendTo(buffer); - } - - @Override - void appendCopyTo(SectionList sectionList) { - sectionList.appendLiteralSection(range().end()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java deleted file mode 100644 index d9e7cdb4ccd..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -/** - * @author hakonhall - */ -public class NameAlreadyExistsTemplateException extends TemplateException { - public NameAlreadyExistsTemplateException(String name, Section first, Section second) { - super("The name '" + name + "' of the " + second.type() + " section at " + - second.range().start().calculateLocation().lineAndColumnText() + - " is in conflict with the identically named " + first.type() + " section at " + - first.range().start().calculateLocation().lineAndColumnText()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java deleted file mode 100644 index a8020cc92d1..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -/** - * @author hakonhall - */ -public class NoSuchNameTemplateException extends TemplateException { - public NoSuchNameTemplateException(CursorRange range, String name) { - super("No such element '" + name + "' in the " + describeSection(range)); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java deleted file mode 100644 index 34879514cd1..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -/** - * @author hakonhall - */ -public class NotBooleanValueTemplateException extends TemplateException { - public NotBooleanValueTemplateException(String name) { - super(name + " was set to a non-boolean value: must be true or false"); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java deleted file mode 100644 index 640baae98ac..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -import java.util.Objects; - -/** - * A section of a template text. - * - * @see Template - * @author hakonhall - */ -abstract class Section { - private final String type; - private final CursorRange range; - private Template template; - - protected Section(String type, CursorRange range) { - this.type = type; - this.range = range; - } - - void setTemplate(Template template) { this.template = template; } - - /** Guaranteed to return non-null after TemplateBuilder::build() returns. */ - protected Template template() { return Objects.requireNonNull(template); } - - protected String type() { return type; } - protected CursorRange range() { return range; } - - abstract void appendTo(StringBuilder buffer); - - abstract void appendCopyTo(SectionList sectionList); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java deleted file mode 100644 index 5a2f5ededc2..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * A mutable list of sections at the same level that can be used to build a template, e.g. the if-body. - * - * @author hakonhall - */ -class SectionList { - private final Cursor start; - private final Cursor end; - private final TemplateBuilder templateBuilder; - - private final List<Section> sections = new ArrayList<>(); - - SectionList(Cursor start, TemplateBuilder templateBuilder) { - this.start = new Cursor(start); - this.end = new Cursor(start); - this.templateBuilder = templateBuilder; - } - - CursorRange range() { return new CursorRange(start, end); } - TemplateBuilder templateBuilder() { return templateBuilder; } - List<Section> sections() { return List.copyOf(sections); } - - void appendLiteralSection(Cursor end) { - CursorRange range = verifyAndUpdateEnd(end); - var section = new LiteralSection(range); - templateBuilder.addLiteralSection(section); - sections.add(section); - } - - VariableSection appendVariableSection(String name, Cursor nameOffset, Cursor end) { - CursorRange range = verifyAndUpdateEnd(end); - var section = new VariableSection(range, name, nameOffset); - templateBuilder.addVariableSection(section); - sections.add(section); - return section; - } - - void appendIfSection(boolean negated, String name, Cursor nameOffset, Cursor end, - SectionList ifSections, Optional<SectionList> elseSections) { - CursorRange range = verifyAndUpdateEnd(end); - var section = new IfSection(range, negated, name, nameOffset, ifSections, elseSections); - templateBuilder.addIfSection(section); - sections.add(section); - } - - ListSection appendListSection(String name, Cursor nameOffset, Cursor end, Template body) { - CursorRange range = verifyAndUpdateEnd(end); - var section = new ListSection(range, name, nameOffset, body); - templateBuilder.addListSection(section); - sections.add(section); - return section; - } - - private CursorRange verifyAndUpdateEnd(Cursor newEnd) { - var range = new CursorRange(this.end, newEnd); - this.end.set(newEnd); - return range; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java deleted file mode 100644 index 818da2d3403..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileWriter; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * The Java representation of a template text. - * - * <p>A template is a sequence of literal text and dynamic sections defined by %{...} directives:</p> - * - * <pre> - * template: section* - * section: literal | variable | list - * literal: plain text not containing %{ - * variable: %{=id} - * if: %{if [!]id}template[%{else}template]%{end} - * list: %{list id}template%{end} - * id: a valid Java identifier - * </pre> - * - * <p>Other directive delimiters than "%{" and "}" may be used, see {@link TemplateDescriptor}.</p> - * - * <p>Fill the template with variable values ({@link #set(String, String) set()}, set if conditions - * ({@link #set(String, boolean)}), add list elements ({@link #add(String) add()}, etc, and finally - * render it as a String ({@link #render()}).</p> - * - * <p>To reuse a template, create the template and work on snapshots of that ({@link #snapshot()}).</p> - * - * @see TemplateDescriptor - * @author hakonhall - */ -public class Template implements Form { - private Template parent = null; - private final CursorRange range; - private final List<Section> sections; - - private final Map<String, String> values = new HashMap<>(); - private final Map<String, ListSection> lists; - - public static Template at(Path path) { return at(path, new TemplateDescriptor()); } - public static Template at(Path path, TemplateDescriptor descriptor) { - String content = new UnixPath(path).readUtf8File(); - return Template.from(content, descriptor); - } - - public static Template from(String text) { return from(text, new TemplateDescriptor()); } - public static Template from(String text, TemplateDescriptor descriptor) { - return TemplateParser.parse(text, descriptor).template(); - } - - Template(CursorRange range, List<Section> sections, Map<String, ListSection> lists) { - this.range = new CursorRange(range); - this.sections = List.copyOf(sections); - this.lists = Map.copyOf(lists); - } - - /** Set the value of a variable, e.g. %{=color}. */ - @Override - public Template set(String name, String value) { - values.put(name, value); - return this; - } - - @Override - public ListElement add(String name) { return new ListElement(addElement(name)); } - - public String render() { - var buffer = new StringBuilder((int) (range.length() * 1.2 + 128)); - appendTo(buffer); - return buffer.toString(); - } - - public void appendTo(StringBuilder buffer) { sections.forEach(section -> section.appendTo(buffer)); } - - /** Returns a deep copy of this. No changes to this affects the returned template, and vice versa. */ - public Template snapshot() { - var builder = new TemplateBuilder(range.start()); - sections.forEach(section -> section.appendCopyTo(builder.topLevelSectionList())); - Template template = builder.build(); - values.forEach(template::set); - return template; - } - - public FileWriter getFileWriterTo(Path path) { - String content = render(); - return new FileWriter(path, () -> content); - } - - /** Must be called (if there is a parent) before any other method. */ - void setParent(Template parent) { this.parent = parent; } - - Template addElement(String name) { - var section = lists.get(name); - if (section == null) { - throw new NoSuchNameTemplateException(range, name); - } - return section.add(); - } - - Optional<String> getVariableValue(String name) { - String value = values.get(name); - if (value != null) return Optional.of(value); - if (parent != null) return parent.getVariableValue(name); - return Optional.empty(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java deleted file mode 100644 index 05b3cce52cc..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author hakonhall - */ -class TemplateBuilder { - /** The top-level section list in this template. */ - private final SectionList sectionList; - private final List<Section> allSections = new ArrayList<>(); - private final Map<String, VariableSection> sampleVariables = new HashMap<>(); - private final Map<String, IfSection> sampleIfSections = new HashMap<>(); - private final Map<String, ListSection> lists = new HashMap<>(); - - TemplateBuilder(Cursor start) { - this.sectionList = new SectionList(start, this); - } - - SectionList topLevelSectionList() { return sectionList; } - - void addLiteralSection(LiteralSection section) { - allSections.add(section); - } - - void addVariableSection(VariableSection section) { - // It's OK if the same name is used in an if-directive (as long as the value is boolean, - // determined when set on a template). - - ListSection existing = lists.get(section.name()); - if (existing != null) - throw new NameAlreadyExistsTemplateException(section.name(), existing, section); - - sampleVariables.put(section.name(), section); - allSections.add(section); - } - - void addIfSection(IfSection section) { - // It's OK if the same name is used in a variable section (as long as the value is boolean, - // determined when set on a template). - - ListSection list = lists.get(section.name()); - if (list != null) - throw new NameAlreadyExistsTemplateException(section.name(), list, section); - - sampleIfSections.put(section.name(), section); - allSections.add(section); - } - - void addListSection(ListSection section) { - VariableSection variableSection = sampleVariables.get(section.name()); - if (variableSection != null) - throw new NameAlreadyExistsTemplateException(section.name(), variableSection, section); - - IfSection ifSection = sampleIfSections.get(section.name()); - if (ifSection != null) - throw new NameAlreadyExistsTemplateException(section.name(), ifSection, section); - - ListSection previous = lists.put(section.name(), section); - if (previous != null) - throw new NameAlreadyExistsTemplateException(section.name(), previous, section); - allSections.add(section); - } - - Template build() { - var template = new Template(sectionList.range(), sectionList.sections(), lists); - allSections.forEach(section -> section.setTemplate(template)); - return template; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java deleted file mode 100644 index 657e30de084..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -/** - * Specifies the how to interpret a template text. - * - * @author hakonhall - */ -public class TemplateDescriptor { - - private String startDelimiter = "%{"; - private String endDelimiter = "}"; - private boolean removeNewline = true; - - public TemplateDescriptor() {} - - public TemplateDescriptor(TemplateDescriptor that) { - this.startDelimiter = that.startDelimiter; - this.endDelimiter = that.endDelimiter; - this.removeNewline = that.removeNewline; - } - - /** Use these delimiters instead of the standard "%{" and "}" to start and end a template directive. */ - public TemplateDescriptor setDelimiters(String startDelimiter, String endDelimiter) { - this.startDelimiter = Token.verifyDelimiter(startDelimiter); - this.endDelimiter = Token.verifyDelimiter(endDelimiter); - return this; - } - - /** - * Whether to remove a newline that immediately follows a non-variable directive. The opposite - * effect can be achieved by preceding the end delimiter with a "-" char, e.g. %{if foo-}. - */ - public TemplateDescriptor setRemoveNewline(boolean removeNewline) { - this.removeNewline = removeNewline; - return this; - } - - public String startDelimiter() { return startDelimiter; } - public String endDelimiter() { return endDelimiter; } - public boolean removeNewline() { return removeNewline; } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java deleted file mode 100644 index 61d13abe4ca..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -/** - * @author hakonhall - */ -public class TemplateException extends RuntimeException { - public TemplateException(String message) { super(message); } - - protected static String describeSection(CursorRange range) { - var startLocation = range.start().calculateLocation(); - var endLocation = range.end().calculateLocation(); - return "template section starting at line " + startLocation.line() + " and column " + startLocation.column() + - ", and ending at line " + endLocation.line() + " and column " + endLocation.column(); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java deleted file mode 100644 index eda5e553576..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; - -/** - * @author hakonhall - */ -public class TemplateNameNotSetException extends TemplateException { - public TemplateNameNotSetException(String name, Cursor nameOffset) { - super("Variable at " + nameOffset.calculateLocation().lineAndColumnText() + " has not been set: " + name); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java deleted file mode 100644 index 814197e80ea..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; - -import java.util.EnumSet; -import java.util.Optional; - -/** - * Parses a template String, see {@link Template} for details. - * - * @author hakonhall - */ -class TemplateParser { - private final TemplateDescriptor descriptor; - private final Cursor start; - private final Cursor current; - private final TemplateBuilder templateBuilder; - - static TemplateParser parse(String text, TemplateDescriptor descriptor) { - return parse(new TemplateDescriptor(descriptor), new Cursor(text), EnumSet.of(Sentinel.EOT)); - } - - private static TemplateParser parse(TemplateDescriptor descriptor, Cursor start, EnumSet<Sentinel> sentinel) { - var parser = new TemplateParser(descriptor, start); - parser.parse(parser.templateBuilder.topLevelSectionList(), sentinel); - return parser; - } - - private enum Sentinel { ELSE, END, EOT } - - private TemplateParser(TemplateDescriptor descriptor, Cursor start) { - this.descriptor = descriptor; - this.start = new Cursor(start); - this.current = new Cursor(start); - this.templateBuilder = new TemplateBuilder(start); - } - - Template template() { return templateBuilder.build(); } - - private Sentinel parse(SectionList sectionList, EnumSet<Sentinel> sentinels) { - do { - current.advanceTo(descriptor.startDelimiter()); - if (!current.equals(start)) { - sectionList.appendLiteralSection(current); - } - - if (current.eot()) { - if (!sentinels.contains(Sentinel.EOT)) { - throw new BadTemplateException(start, "Missing end directive for section started"); - } - return Sentinel.EOT; - } - - Optional<Sentinel> sentinel = parseSection(sectionList, sentinels); - if (sentinel.isPresent()) return sentinel.get(); - } while (true); - } - - private Optional<Sentinel> parseSection(SectionList sectionList, EnumSet<Sentinel> sentinels) { - current.skip(descriptor.startDelimiter()); - - if (current.skip(Token.VARIABLE_DIRECTIVE_CHAR)) { - parseVariableSection(sectionList); - } else { - var startOfType = new Cursor(current); - String type = skipId().orElseThrow(() -> new BadTemplateException(current, "Missing section name")); - - switch (type) { - case "else" -> { - if (!sentinels.contains(Sentinel.ELSE)) - throw new BadTemplateException(startOfType, "Stray 'else'"); - parseEndDirective(); - return Optional.of(Sentinel.ELSE); - } - case "end" -> { - if (!sentinels.contains(Sentinel.END)) - throw new BadTemplateException(startOfType, "Stray 'end'"); - parseEndDirective(); - return Optional.of(Sentinel.END); - } - case "if" -> parseIfSection(sectionList); - case "list" -> parseListSection(sectionList); - default -> throw new BadTemplateException(startOfType, "Unknown section '" + type + "'"); - } - } - - return Optional.empty(); - } - - private void parseVariableSection(SectionList sectionList) { - var nameStart = new Cursor(current); - String name = parseId(); - parseEndDelimiter(false); - sectionList.appendVariableSection(name, nameStart, current); - } - - private void parseEndDirective() { - parseEndDelimiter(true); - } - - private void parseListSection(SectionList sectionList) { - skipRequiredWhitespaces(); - var startOfName = new Cursor(current); - String name = parseId(); - parseEndDelimiter(true); - - TemplateParser bodyParser = parse(descriptor, current, EnumSet.of(Sentinel.END)); - current.set(bodyParser.current); - - sectionList.appendListSection(name, startOfName, current, bodyParser.templateBuilder.build()); - } - - private void parseIfSection(SectionList sectionList) { - skipRequiredWhitespaces(); - boolean negated = current.skip(Token.NEGATE_CHAR); - current.skipWhitespaces(); - var startOfName = new Cursor(current); - String name = parseId(); - parseEndDelimiter(true); - - SectionList ifSectionList = new SectionList(current, templateBuilder); - Sentinel ifSentinel = parse(ifSectionList, EnumSet.of(Sentinel.ELSE, Sentinel.END)); - - Optional<SectionList> elseSectionList = Optional.empty(); - if (ifSentinel == Sentinel.ELSE) { - elseSectionList = Optional.of(new SectionList(current, templateBuilder)); - parse(elseSectionList.get(), EnumSet.of(Sentinel.END)); - } - - sectionList.appendIfSection(negated, name, startOfName, current, ifSectionList, elseSectionList); - } - - private void skipRequiredWhitespaces() { - if (!current.skipWhitespaces()) { - throw new BadTemplateException(current, "Expected whitespace"); - } - } - - private String parseId() { - return skipId().orElseThrow(() -> new BadTemplateException(current, "Expected identifier")); - } - - private Optional<String> skipId() { return Token.skipId(current); } - - private void parseEndDelimiter(boolean allowSkipNewline) { - boolean removeNewlineCharPresent = current.skip(Token.REMOVE_NEWLINE_CHAR); - - if (!current.skip(descriptor.endDelimiter())) - throw new BadTemplateException(current, "Expected section end (" + descriptor.endDelimiter() + ")"); - - // The presence of the remove-newline-char means the opposite behavior is wanted. - if (allowSkipNewline && (removeNewlineCharPresent != descriptor.removeNewline())) - current.skip('\n'); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java deleted file mode 100644 index 61e9b27372c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -import java.util.Optional; - -/** - * @author hakonhall - */ -class Token { - static final char NEGATE_CHAR = '!'; - static final char REMOVE_NEWLINE_CHAR = '-'; - static final char VARIABLE_DIRECTIVE_CHAR = '='; - - static Optional<String> skipId(Cursor cursor) { - if (cursor.eot() || !isIdStart(cursor.getChar())) return Optional.empty(); - - Cursor start = new Cursor(cursor); - cursor.increment(); - - while (!cursor.eot() && isIdPart(cursor.getChar())) - cursor.increment(); - - return Optional.of(new CursorRange(start, cursor).string()); - } - - /** A delimiter either starts a directive (e.g. %{) or ends it (e.g. }). */ - static String verifyDelimiter(String delimiter) { - if (!isAsciiToken(delimiter)) { - throw new IllegalArgumentException("Invalid delimiter: '" + delimiter + "'"); - } - return delimiter; - } - - /** Returns true for a non-empty string with only ASCII token characters. */ - private static boolean isAsciiToken(String string) { - if (string.isEmpty()) return false; - for (char c : string.toCharArray()) { - if (!isAsciiTokenChar(c)) return false; - } - return true; - } - - /** Returns true if char is a printable ASCII character except space (isgraph(3)). */ - private static boolean isAsciiTokenChar(char c) { - // 0x1F unit separator - // 0x20 space - // 0x21 ! - // ... - // 0x7E ~ - // 0x7F del - return 0x20 < c && c < 0x7F; - } - - // Our identifiers are equivalent to a Java identifiers. - private static boolean isIdStart(char c) { return Character.isJavaIdentifierStart(c); } - private static boolean isIdPart(char c) { return Character.isJavaIdentifierPart(c); } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java deleted file mode 100644 index 1423e9774af..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - -/** - * Represents a template variable section - * - * @see Template - * @author hakonhall - */ -class VariableSection extends Section { - private final String name; - private final Cursor nameOffset; - - VariableSection(CursorRange range, String name, Cursor nameOffset) { - super("variable", range); - this.name = name; - this.nameOffset = nameOffset; - } - - String name() { return name; } - Cursor nameOffset() { return new Cursor(nameOffset); } - - @Override - void appendTo(StringBuilder buffer) { - String value = template().getVariableValue(name) - .orElseThrow(() -> new TemplateNameNotSetException(name, nameOffset)); - buffer.append(value); - } - - @Override - void appendCopyTo(SectionList sectionList) { - sectionList.appendVariableSection(name, nameOffset, range().end()); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java deleted file mode 100644 index 0618b0c09e5..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java deleted file mode 100644 index f6f2005dcab..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.text; - -import java.util.Objects; - -/** - * Cursor is a mutable offset into a fixed String, and useful for String parsing. - * - * @author hakonhall - */ -// @Mutable -public class Cursor { - private final String text; - private int offset; - private TextLocation locationCache; - - /** Creates a pointer to the first char of {@code text}, which is EOT if {@code text} is empty. */ - public Cursor(String text) { this(text, 0, new TextLocation()); } - - public Cursor(Cursor that) { this(that.text, that.offset, that.locationCache); } - - private Cursor(String text, int offset, TextLocation location) { - this.text = Objects.requireNonNull(text); - this.offset = offset; - this.locationCache = Objects.requireNonNull(location); - } - - /** Returns the substring of {@code text} starting at {@link #offset()} (to EOT). */ - @Override - public String toString() { return text.substring(offset); } - - public String fullText() { return text; } - public int offset() { return offset; } - public boolean bot() { return offset == 0; } - public boolean eot() { return offset == text.length(); } - public boolean startsWith(char c) { return offset < text.length() && text.charAt(offset) == c; } - public boolean startsWith(String prefix) { return text.startsWith(prefix, offset); } - - /** @throws IndexOutOfBoundsException if {@link #eot()}. */ - public char getChar() { return text.charAt(offset); } - - /** The number of chars between pointer and EOT. */ - public int length() { return text.length() - offset; } - - /** Calculate the current text location in O(length(text)). */ - public TextLocation calculateLocation() { - if (offset < locationCache.offset()) { - locationCache = new TextLocation(); - } else if (offset == locationCache.offset()) { - return locationCache; - } - - int lineIndex = locationCache.lineIndex(); - int columnIndex = locationCache.columnIndex(); - for (int i = locationCache.offset(); i < offset; ++i) { - if (text.charAt(i) == '\n') { - ++lineIndex; - columnIndex = 0; - } else { - ++columnIndex; - } - } - - locationCache = new TextLocation(offset, lineIndex, columnIndex); - return locationCache; - } - - public void set(Cursor that) { - if (that.text != text) { - throw new IllegalArgumentException("'that' doesn't refer to the same text"); - } - - this.offset = that.offset; - } - - /** Advance substring.length() if this startsWith the substring, returning true if so. */ - public boolean skip(String substring) { - if (startsWith(substring)) { - offset += substring.length(); - return true; - } else { - return false; - } - } - - public boolean skip(char c) { - if (startsWith(c)) { - ++offset; - return true; - } else { - return false; - } - } - - /** If the current char is a whitespace, skip it and return true. */ - public boolean skipWhitespace() { - if (!eot() && Character.isWhitespace(getChar())) { - ++offset; - return true; - } else { - return false; - } - } - - /** Returns true if at least one whitespace was skipped. */ - public boolean skipWhitespaces() { - if (skipWhitespace()) { - while (skipWhitespace()) - ++offset; - return true; - } else { - return false; - } - } - - /** Return false if eot(), otherwise advance to the next char and return true. */ - public boolean increment() { - if (eot()) return false; - ++offset; - return true; - } - - /** - * Advance {@code distance} chars until bot() or eot() is reached (distance may be negative), - * and return true if this cursor moved the full distance. - */ - public boolean advance(int distance) { - int newOffset = offset + distance; - if (newOffset < 0) { - this.offset = 0; - return false; - } else if (newOffset > text.length()) { - this.offset = text.length(); - return false; - } else { - this.offset = newOffset; - return true; - } - } - - /** Advance pointer until start of needle is found (and return true), or EOT is reached (and return false). */ - public boolean advanceTo(String needle) { - int index = text.indexOf(needle, offset); - if (index == -1) { - offset = text.length(); - return false; // and eot() is true - } else { - offset = index; - return true; // and eot() is false - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Cursor cursor = (Cursor) o; - return offset == cursor.offset && text.equals(cursor.text); - } - - @Override - public int hashCode() { - return Objects.hash(text, offset); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java deleted file mode 100644 index b70aabbb1ec..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.text; - -/** - * A start- and end- offset in an underlying String. - * - * @author hakonhall - */ -public class CursorRange { - private final Cursor start; - private final Cursor end; - - @SuppressWarnings("StringEquality") - public CursorRange(Cursor start, Cursor end) { - if (start.fullText() != end.fullText()) { - throw new IllegalArgumentException("start and end points to different texts"); - } - - if (start.offset() > end.offset()) { - throw new IllegalArgumentException("start offset " + start.offset() + - " is beyond end offset " + end.offset()); - } - - this.start = new Cursor(start); - this.end = new Cursor(end); - } - - public CursorRange(CursorRange that) { - this.start = new Cursor(that.start); - this.end = new Cursor(that.end); - } - - public Cursor start() { return new Cursor(start); } - public Cursor end() { return new Cursor(end); } - public int length() { return end.offset() - start.offset(); } - public String string() { return start.fullText().substring(start.offset(), end.offset()); } - public void appendTo(StringBuilder buffer) { buffer.append(start.fullText(), start.offset(), end.offset()); } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java deleted file mode 100644 index 3d47a782103..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.text; - -/** - * The location within an implied multi-line String. - * - * @author hakonhall - */ -//@Immutable -public class TextLocation { - private final int offset; - private final int lineIndex; - private final int columnIndex; - - public TextLocation() { this(0, 0, 0); } - - public TextLocation(int offset, int lineIndex, int columnIndex) { - this.offset = offset; - this.lineIndex = lineIndex; - this.columnIndex = columnIndex; - } - - public int offset() { return offset; } - public int lineIndex() { return lineIndex; } - public int line() { return lineIndex + 1; } - public int columnIndex() { return columnIndex; } - public int column() { return columnIndex + 1; } - - public String lineAndColumnText() { return "line " + line() + " and column " + column(); } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/package-info.java deleted file mode 100644 index efda4639579..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util.text; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java deleted file mode 100644 index de89ad1489d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; - -import java.util.List; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.node.admin.task.util.yum.YumCommand.DeleteVersionLockYumCommand; -import static com.yahoo.vespa.hosted.node.admin.task.util.yum.YumCommand.GenericYumCommand; -import static com.yahoo.vespa.hosted.node.admin.task.util.yum.YumCommand.InstallFixedYumCommand; - -/** - * @author hakonhall - */ -public class Yum { - - private final Terminal terminal; - - public Yum(Terminal terminal) { - this.terminal = terminal; - } - - public Optional<YumPackageName> queryInstalled(TaskContext context, String packageName) { - return YumCommand.queryInstalled(terminal, context, YumPackageName.fromString(packageName)); - } - - /** Lock and install, or if necessary downgrade, a package to a given version. */ - public InstallFixedYumCommand installFixedVersion(YumPackageName yumPackage) { - return new InstallFixedYumCommand(terminal, yumPackage); - } - - public GenericYumCommand install(YumPackageName... packages) { - return new GenericYumCommand(terminal, GenericYumCommand.CommandType.install, List.of(packages)); - } - - public GenericYumCommand install(String package1, String... packages) { - return install(toYumPackageNameArray(package1, packages)); - } - - public GenericYumCommand install(List<String> packages) { - return install(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new)); - } - - - public GenericYumCommand upgrade(YumPackageName... packages) { - return new GenericYumCommand(terminal, GenericYumCommand.CommandType.upgrade, List.of(packages)); - } - - public GenericYumCommand upgrade(String package1, String... packages) { - return upgrade(toYumPackageNameArray(package1, packages)); - } - - public GenericYumCommand upgrade(List<String> packages) { - return upgrade(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new)); - } - - - public GenericYumCommand remove(YumPackageName... packages) { - return new GenericYumCommand(terminal, GenericYumCommand.CommandType.remove, List.of(packages)); - } - - public GenericYumCommand remove(String package1, String... packages) { - return remove(toYumPackageNameArray(package1, packages)); - } - - public GenericYumCommand remove(List<String> packages) { - return remove(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new)); - } - - public YumCommand.DeleteVersionLockYumCommand deleteVersionLock(YumPackageName yumPackage) { - return new DeleteVersionLockYumCommand(terminal, yumPackage); - } - - static YumPackageName[] toYumPackageNameArray(String package1, String... packages) { - YumPackageName[] array = new YumPackageName[1 + packages.length]; - array[0] = YumPackageName.fromString(package1); - for (int i = 0; i < packages.length; ++i) { - array[1 + i] = YumPackageName.fromString(packages[i]); - } - return array; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java deleted file mode 100644 index 64b637fdd5b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -/** - * @author freva - */ -public abstract class YumCommand<T extends YumCommand<T>> { - - // Note: "(?dm)" makes newline be \n (only), and enables multiline mode where ^$ match lines with find() - public static final Pattern INSTALL_NOOP_PATTERN = Pattern.compile("(?dm)^Nothing to do\\.?$"); - public static final Pattern UPGRADE_NOOP_PATTERN = Pattern.compile("(?dm)^No packages marked for update$"); - public static final Pattern REMOVE_NOOP_PATTERN = Pattern.compile("(?dm)^No [pP]ackages marked for removal\\.?$"); - - // WARNING: These must be in the same order as the supplier below - private static final String RPM_QUERYFORMAT = Stream.of("NAME", "EPOCH", "VERSION", "RELEASE", "ARCH") - .map(formatter -> "%{" + formatter + "}") - .collect(Collectors.joining("\\n")); - private static final Function<YumPackageName.Builder, List<Function<String, YumPackageName.Builder>>> - PACKAGE_NAME_BUILDERS_GENERATOR = builder -> List.of( - builder::setName, builder::setEpoch, builder::setVersion, builder::setRelease, builder::setArchitecture); - - private List<String> disabledRepos = List.of(); - private List<String> enabledRepos = List.of(); - private final Terminal terminal; - - protected YumCommand(Terminal terminal) { - this.terminal = terminal; - } - - /** Disables the given repos for this command */ - public T disableRepo(String... repo) { - disabledRepos = List.of(repo); - return getThis(); - } - - /** Enables the given repos for this command */ - public T enableRepo(String... repo) { - enabledRepos = List.of(repo); - return getThis(); - } - - protected abstract T getThis(); // Hack to get around unchecked cast warning - - protected void addParametersToCommandLine(CommandLine commandLine) { - commandLine.add("--assumeyes"); - disabledRepos.forEach(repo -> commandLine.add("--disablerepo=" + repo)); - enabledRepos.forEach(repo -> commandLine.add("--enablerepo=" + repo)); - } - - public abstract boolean converge(TaskContext context); - - public static class GenericYumCommand extends YumCommand<GenericYumCommand> { - private static final Pattern UNKNOWN_PACKAGE_PATTERN = Pattern.compile("(?dm)^No package ([^ ]+) available\\.$"); - - private final Terminal terminal; - private final CommandType yumCommand; - private final List<YumPackageName> packages; - private final List<String> options = new ArrayList<>(); - - GenericYumCommand(Terminal terminal, CommandType yumCommand, List<YumPackageName> packages) { - super(terminal); - this.terminal = terminal; - this.yumCommand = yumCommand; - this.packages = packages; - - switch (yumCommand) { - case install: { - if (packages.size() > 1) options.add("skip_missing_names_on_install=False"); - break; - } - case upgrade: { - if (packages.size() > 1) options.add("skip_missing_names_on_update=False"); - break; - } - case remove: break; - default: throw new IllegalArgumentException("Unknown yum command: " + yumCommand); - } - - if (packages.isEmpty() && yumCommand != CommandType.upgrade) - throw new IllegalArgumentException("No packages specified"); - } - - @Override - protected void addParametersToCommandLine(CommandLine commandLine) { - super.addParametersToCommandLine(commandLine); - options.forEach(option -> commandLine.add("--setopt", option)); - } - - @Override - public boolean converge(TaskContext context) { - if (yumCommand == CommandType.install) - if (packages.stream().allMatch(pkg -> isInstalled(context, pkg))) return false; - if (yumCommand == CommandType.remove) - if (packages.stream().noneMatch(pkg -> isInstalled(context, pkg))) return false; - - CommandLine commandLine = terminal.newCommandLine(context); - commandLine.add("yum", yumCommand.name()); - addParametersToCommandLine(commandLine); - commandLine.add(packages.stream().map(pkg -> pkg.toName()).toList()); - - // There's no way to figure out whether a yum command would have been a no-op. - // Therefore, run the command and parse the output to decide. - boolean modifiedSystem = commandLine - .executeSilently() - .mapOutput(this::packageChanged); - - if (modifiedSystem) { - commandLine.recordSilentExecutionAsSystemModification(); - } - - return modifiedSystem; - } - - private boolean packageChanged(String output) { - Matcher unknownPackageMatcher = UNKNOWN_PACKAGE_PATTERN.matcher(output); - if (unknownPackageMatcher.find()) { - throw new IllegalArgumentException("Unknown package: " + unknownPackageMatcher.group(1)); - } - - return yumCommand.outputNoopPatterns.stream().noneMatch(pattern -> pattern.matcher(output).find()); - } - - protected GenericYumCommand getThis() { return this; } - - enum CommandType { - install(INSTALL_NOOP_PATTERN), remove(REMOVE_NOOP_PATTERN), upgrade(INSTALL_NOOP_PATTERN, UPGRADE_NOOP_PATTERN); - - private final List<Pattern> outputNoopPatterns; - CommandType(Pattern... outputNoopPatterns) { - this.outputNoopPatterns = List.of(outputNoopPatterns); - } - } - } - - - public static class InstallFixedYumCommand extends YumCommand<InstallFixedYumCommand> { - // Note: "(?dm)" makes newline be \n (only), and enables multiline mode where ^$ match lines with find() - private static final Pattern CHECKING_FOR_UPDATE_PATTERN = - Pattern.compile("(?dm)^Package matching [^ ]+ already installed\\. Checking for update\\.$"); - - private final Terminal terminal; - private final YumPackageName yumPackage; - - InstallFixedYumCommand(Terminal terminal, YumPackageName yumPackage) { - super(terminal); - this.terminal = terminal; - this.yumPackage = yumPackage; - } - - @Override - public boolean converge(TaskContext context) { - String targetVersionLockName = yumPackage.toVersionLockName(); - - boolean alreadyLocked = false; - Optional<String> versionLock = versionLockExists(context, terminal, yumPackage); - if (versionLock.isPresent()) { - if (versionLock.get().equals(targetVersionLockName)) { - alreadyLocked = true; - } else { - YumCommand.deleteVersionLock(context, terminal, versionLock.get()); - } - } - - boolean modified = false; - - if (!alreadyLocked) { - CommandLine commandLine = terminal.newCommandLine(context).add("yum", "versionlock", "add"); - // If the targetVersionLockName refers to a package in a by-default-disabled repo, - // we must enable the repo unless targetVersionLockName is already installed. - // The other versionlock commands (list, delete) does not require --enablerepo. - addParametersToCommandLine(commandLine); - commandLine.add(targetVersionLockName).execute(); - modified = true; - } - - // The following 3 things may happen with yum install: - // 1. The package is installed or upgraded to the target version, in case we'd return - // true from converge() - // 2. The package is already installed at target version, in case - // "Nothing to do" is printed in the last line and we may return false from converge() - // 3. The package is already installed but at a later version than the target version, - // in case the last 2 lines of the output is: - // - "Package matching yakl-client-0.10-654.el7.x86_64 already installed. Checking for update." - // - "Nothing to do" - // And in case we need to downgrade and return true from converge() - - var installCommand = terminal.newCommandLine(context).add("yum", "install"); - addParametersToCommandLine(installCommand); - installCommand.add(yumPackage.toName()); - - String output = installCommand.executeSilently().getUntrimmedOutput(); - - if (INSTALL_NOOP_PATTERN.matcher(output).find()) { - if (CHECKING_FOR_UPDATE_PATTERN.matcher(output).find()) { - // case 3. - var upgradeCommand = terminal.newCommandLine(context).add("yum", "downgrade"); - addParametersToCommandLine(upgradeCommand); - upgradeCommand.add(yumPackage.toName()).execute(); - modified = true; - } else { - // case 2. - } - } else { - // case 1. - installCommand.recordSilentExecutionAsSystemModification(); - modified = true; - } - - return modified; - } - - protected InstallFixedYumCommand getThis() { return this; } - } - - public static class DeleteVersionLockYumCommand extends YumCommand<DeleteVersionLockYumCommand> { - private final Terminal terminal; - private final YumPackageName yumPackage; - - DeleteVersionLockYumCommand(Terminal terminal, YumPackageName yumPackage) { - super(terminal); - this.terminal = terminal; - this.yumPackage = yumPackage; - } - - @Override - public boolean converge(TaskContext context) { - return deleteVersionLock(context, terminal, yumPackage.toName()); - } - - protected DeleteVersionLockYumCommand getThis() { return this; } - } - - protected boolean isInstalled(TaskContext context, YumPackageName yumPackage) { - return queryInstalled(terminal, context, yumPackage).map(yumPackage::isSubsetOf).orElse(false); - } - - static Optional<YumPackageName> queryInstalled(Terminal terminal, TaskContext context, YumPackageName yumPackage) { - String packageName = yumPackage.toName(); - CommandResult commandResult = terminal.newCommandLine(context) - .add("rpm", "-q", packageName, "--queryformat", RPM_QUERYFORMAT) - .ignoreExitCode() - .executeSilently(); - - if (commandResult.getExitCode() != 0) return Optional.empty(); - - YumPackageName.Builder builder = new YumPackageName.Builder(); - List<Function<String, YumPackageName.Builder>> builders = PACKAGE_NAME_BUILDERS_GENERATOR.apply(builder); - List<Optional<String>> lines = commandResult.mapEachLine(line -> Optional.of(line).filter(s -> !"(none)".equals(s))); - if (lines.size() % builders.size() != 0) throw new IllegalStateException(String.format("Unexpected response from rpm, expected %d lines, got '%s'", builders.size(), commandResult.getOutput())); - if (lines.size() > builders.size()) throw new IllegalArgumentException("Found multiple installed packages for '" + packageName + "'. Version is required to match package exactly"); - - IntStream.range(0, builders.size()).forEach(i -> lines.get(i).ifPresent(builders.get(i)::apply)); - if (builder.epoch().isEmpty()) builder.setEpoch("0"); - - return Optional.of(builder.build()); - } - - private static Optional<String> versionLockExists(TaskContext context, Terminal terminal, YumPackageName yumPackage) { - - List<String> command = new ArrayList<>(4); - command.add("yum"); - command.add("versionlock"); - command.add("list"); - - return terminal - .newCommandLine(context) - .add(command) - .executeSilently() - .getOutputLinesStream() - .map(YumPackageName::parseString) - .filter(Optional::isPresent) // removes garbage first lines, even with --quiet - .map(Optional::get) - // Ignore lines for other packages - .filter(packageName -> packageName.getName().equals(yumPackage.getName())) - // If existing lock doesn't exactly match the full package name, - // it means it's locked to another version and we must remove that lock. - .map(YumPackageName::toVersionLockName) - .findFirst(); - } - - private static boolean deleteVersionLock(TaskContext context, Terminal terminal, String wildcardEntry) { - // Idempotent command, gives exit code 0 also when versionlock does not exist - terminal.newCommandLine(context) - .add("yum", "versionlock", "delete", wildcardEntry) - .execute() - .getOutputLinesStream(); - return true; - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java deleted file mode 100644 index 7fac5c57d06..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.google.common.base.Strings; - -import java.util.Arrays; -import java.util.Objects; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * A YUM/DNF package name. - * - * <p>From yum(8): YUM package names are used with install, update, remove, list, info etc - * with any of the following as well as globs of any of the following, with any of the - * following as well as globs of any of the following: - * - * <ol> - * <li>name - * <li>name.arch - * <li>name-ver - * <li>name-ver-rel - * <li>name-ver-rel.arch - * <li>name-epoch:ver-rel.arch - * <li>epoch:name-ver-rel.arch - * </ol> - * - * <p>However this specification is terribly ambiguous. This class allows constructing - * a package name from its components, which is beneficial because with certain YUM - * commands that needs to canonicalize names (e.g. versionlock). - * - * @author hakonhall - */ -public class YumPackageName { - - private enum Architecture { noarch, x86_64, i386, i586, i686 } - - private static final String ARCHITECTURES_OR = - Arrays.stream(Architecture.values()).map(Architecture::name).collect(Collectors.joining("|")); - private static final Pattern ARCHITECTURE_PATTERN = Pattern.compile("\\.(" + ARCHITECTURES_OR + "|\\*)$"); - private static final Pattern EPOCH_PATTERN = Pattern.compile("^((.+)-)?([0-9]+)$"); - private static final Pattern NAME_VER_REL_PATTERN = - Pattern.compile("^(.+?)?" + // name - "-([+()a-z0-9._]*[0-9]+\\.[a-z0-9._]*)" + // ver: contains at least one digit and dot - "(?:-([+()a-z0-9._]*[0-9][a-z0-9._]*))?$"); // rel: contains at least one digit - private static final Pattern NAME_PATTERN = Pattern.compile("^[+()a-zA-Z0-9._-]+$"); - - private final Optional<String> epoch; - private final String name; - private final Optional<String> version; - private final Optional<String> release; - private final Optional<String> architecture; - - public static class Builder { - private Optional<String> epoch = Optional.empty(); - private String name; - private Optional<String> version = Optional.empty(); - private Optional<String> release = Optional.empty(); - private Optional<String> architecture = Optional.empty(); - - public Builder() { } - - public Builder(String name) { - this.name = name; - } - - public Builder(YumPackageName packageName) { - epoch = packageName.epoch; - name = packageName.name; - version = packageName.version; - release = packageName.release; - architecture = packageName.architecture; - } - - public Builder setEpoch(String epoch) { this.epoch = Optional.of(epoch); return this; } - public Builder setName(String name) { this.name = name; return this; } - public Builder setVersion(String version) { this.version = Optional.of(version); return this; } - public Builder setRelease(String release) { this.release = Optional.of(release); return this; } - public Builder setArchitecture(String architecture) { this.architecture = Optional.of(architecture); return this; } - - public Optional<String> epoch() { return epoch; } - public String name() { return name; } - public Optional<String> version() { return version; } - public Optional<String> release() { return release; } - public Optional<String> architecture() { return architecture; } - - public YumPackageName build() { return new YumPackageName(epoch, name, version, release, architecture); } - } - - /** @see Builder */ - private YumPackageName(Optional<String> epoch, - String name, - Optional<String> version, - Optional<String> release, - Optional<String> architecture) { - if (Strings.isNullOrEmpty(name)) - throw new IllegalArgumentException("name cannot be null or empty"); - this.epoch = epoch; - this.name = name; - this.version = version; - this.release = release; - this.architecture = architecture; - } - - /** - * Parse the string specification of a YUM package. - * - * <p>The following formats are supported: - * - * <ol> - * <li>name - * <li>name.arch - * <li>name-ver - * <li>name-ver-rel - * <li>name-ver-rel.arch - * <li>name-epoch:ver-rel.arch - * <li>epoch:name-ver-rel.arch - * </ol> - * - * @throws IllegalArgumentException if spec does not specify a package name. - * @see #parseString(String) - */ - public static YumPackageName fromString(String packageSpec) { - String spec = packageSpec; - Optional<String> epoch = Optional.empty(); - String name = null; - - // Parse epoch and remove it from spec - int epochColon = spec.indexOf(':'); - if (epochColon >= 0) { - Matcher epochMatcher = EPOCH_PATTERN.matcher(spec.substring(0, epochColon)); - if (!epochMatcher.find()) { - throw new IllegalArgumentException("Unexpected epoch format: " + packageSpec); - } - - name = epochMatcher.group(2); - epoch = Optional.of(epochMatcher.group(3)); - - spec = epochColon == 0 - ? spec.substring(epochColon + 1) - : spec.substring(0, epochColon - 1) + spec.substring(epochColon + 1); - } - - // Parse architecture and remove it from spec - Optional<String> architecture = Optional.empty(); - Matcher architectureMatcher = ARCHITECTURE_PATTERN.matcher(spec); - if (architectureMatcher.find()) { - architecture = Optional.of(architectureMatcher.group(1)); - spec = spec.substring(0, architectureMatcher.start()); - } - - // Parse name, version and release and remove the latter two from spec - Optional<String> version = Optional.empty(); - Optional<String> release = Optional.empty(); - Matcher matcher = NAME_VER_REL_PATTERN.matcher(spec); - if (matcher.find()) { - spec = matcher.group(1); - if (spec == null) { - if (name == null) { - throw new IllegalArgumentException("No package name was found: " + packageSpec); - } - spec = name; // makes spec hold the package name in all cases below. - } - - version = Optional.of(matcher.group(2)); - release = Optional.ofNullable(matcher.group(3)); - } - - if (!NAME_PATTERN.matcher(spec).find()) { - throw new IllegalArgumentException("Bad package name in " + packageSpec + ": '" + spec + "'"); - } - name = spec; - - return new YumPackageName(epoch, name, version, release, architecture); - } - - /** See {@link #fromString(String)}. */ - public static Optional<YumPackageName> parseString(final String packageSpec) { - try { - return Optional.of(fromString(packageSpec)); - } catch (IllegalArgumentException e) { - return Optional.empty(); - } - } - - public Optional<String> getEpoch() { return epoch; } - public String getName() { return name; } - public Optional<String> getVersion() { return version; } - public Optional<String> getRelease() { return release; } - public Optional<String> getArchitecture() { return architecture; } - - /** Return package name, omitting components that are not specified. */ - public String toName() { - StringBuilder builder = new StringBuilder(); - char nextDelimiter; - builder.append(name); - // Fully versioned package names must always include epoch in Yum 4 - epoch.or(() -> Optional.of("0").filter(v -> version.isPresent())) - .ifPresent(ep -> builder.append('-').append(ep)); - nextDelimiter = ':'; - version.ifPresent(s -> builder.append(nextDelimiter).append(s)); - release.ifPresent(s -> builder.append('-').append(s)); - architecture.ifPresent(arch -> builder.append('.').append(arch)); - return builder.toString(); - } - - /** - * The package name output by 'yum versionlock list'. Can also be used with 'add' and 'delete'. - * - * @throws IllegalStateException if any field required for the version lock spec is missing - */ - public String toVersionLockName() { - YumPackageName lockSpec = new Builder(this).setArchitecture("*").build(); - if (lockSpec.getVersion().isEmpty()) throw new IllegalStateException("Version is missing for YUM package " + name); - if (lockSpec.getRelease().isEmpty()) throw new IllegalStateException("Release is missing for YUM package " + name); - return lockSpec.toName(); - } - - public boolean isSubsetOf(YumPackageName other) { - return Objects.equals(name, other.name) && - (epoch.isEmpty() || Objects.equals(epoch, other.epoch)) && - (version.isEmpty() || Objects.equals(version, other.version)) && - (release.isEmpty() || Objects.equals(release, other.release)) && - (architecture.isEmpty() || Objects.equals(architecture, other.architecture)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - YumPackageName that = (YumPackageName) o; - return Objects.equals(epoch, that.epoch) && - Objects.equals(name, that.name) && - Objects.equals(version, that.version) && - Objects.equals(release, that.release) && - Objects.equals(architecture, that.architecture); - } - - @Override - public int hashCode() { - return Objects.hash(epoch, name, version, release, architecture); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java deleted file mode 100644 index d4fc670e43a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * A {@link Yum} tester that simplifies testing interaction with yum. - * - * @author freva - */ -public class YumTester extends Yum { - - private final TestTerminal terminal; - - public YumTester(TestTerminal terminal) { - super(terminal); - this.terminal = terminal; - } - - public GenericYumCommandExpectation expectInstall(String... packages) { - return new GenericYumCommandExpectation(CommandType.install, packages); - } - - public GenericYumCommandExpectation expectUpdate(String... packages) { - return new GenericYumCommandExpectation(CommandType.upgrade, packages); - } - - public GenericYumCommandExpectation expectRemove(String... packages) { - return new GenericYumCommandExpectation(CommandType.remove, packages); - } - - public InstallFixedCommandExpectation expectInstallFixedVersion(String yumPackage) { - return new InstallFixedCommandExpectation(yumPackage); - } - - public DeleteVersionLockCommandExpectation expectDeleteVersionLock(String yumPackage) { - return new DeleteVersionLockCommandExpectation(yumPackage); - } - - public QueryInstalledExpectation expectQueryInstalled(String packageName) { - return new QueryInstalledExpectation(packageName); - } - - - public class GenericYumCommandExpectation { - private final CommandType commandType; - protected final List<YumPackageName> packages; - private List<String> disableRepos = List.of(); - private List<String> enableRepos = List.of(); - - private GenericYumCommandExpectation(CommandType commandType, String... packages) { - this.commandType = commandType; - this.packages = Stream.of(packages).map(YumPackageName::fromString).toList(); - } - - public GenericYumCommandExpectation withDisableRepo(String... repo) { - this.disableRepos = List.of(repo); - return this; - } - - public GenericYumCommandExpectation withEnableRepo(String... repo) { - this.enableRepos = List.of(repo); - return this; - } - - /** Mock the return value of the converge(TaskContext) method for this operation (true iff system was modified) */ - public YumTester andReturn(boolean value) { - if (value) return execute("Success"); - return switch (commandType) { - case deleteVersionLock, installFixed, install -> execute("Nothing to do"); - case upgrade -> execute("No packages marked for update"); - case remove -> execute("No Packages marked for removal"); - default -> throw new IllegalArgumentException("Unknown command type: " + commandType); - }; - } - - private YumTester execute(String output) { - if (commandType == CommandType.install) - terminal.interceptCommand("rpm query", cmd -> new TestChildProcess2(1, "Not installed")); - if (commandType == CommandType.remove) { // Pretend the first package is installed so we can continue to yum commands - YumPackageName pkg = packages.get(0); - terminal.interceptCommand("rpm query", cmd -> new TestChildProcess2(0, String.join("\n", - pkg.getName(), - pkg.getEpoch().orElse("(none)"), - pkg.getVersion().orElse("1.2.3"), - pkg.getRelease().orElse("1"), - pkg.getArchitecture().orElse("(none)")))); - } - - StringBuilder cmd = new StringBuilder(); - cmd.append("yum ").append(commandType.command); - if (commandType != CommandType.deleteVersionLock) { - cmd.append(" --assumeyes"); - // mimic maybeEscapeArgument() in CommandLine - disableRepos.forEach(repo -> { - if (repo.equals("*")) cmd.append(" \"--disablerepo=").append(repo).append("\""); - else cmd.append(" --disablerepo=").append(repo); - }); - enableRepos.forEach(repo -> cmd.append(" --enablerepo=").append(repo)); - } - if (commandType == CommandType.install && packages.size() > 1) - cmd.append(" --setopt skip_missing_names_on_install=False"); - if (commandType == CommandType.upgrade && packages.size() > 1) - cmd.append(" --setopt skip_missing_names_on_update=False"); - packages.forEach(pkg -> { - String name = pkg.toName(); - if (name.contains("(") || name.contains(")")) { // Ugly hack to handle implicit quoting done in com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine - name = "\"" + name + "\""; - } - cmd.append(" ").append(name); - }); - cmd.append(" 2>&1"); - - terminal.expectCommand(cmd.toString(), 0, output); - return YumTester.this; - } - } - - public class InstallFixedCommandExpectation extends GenericYumCommandExpectation { - - private InstallFixedCommandExpectation(String yumPackage) { - super(CommandType.installFixed, yumPackage); - } - - @Override - public YumTester andReturn(boolean value) { - terminal.expectCommand("yum versionlock list 2>&1", 0, packages.get(0).toVersionLockName()); - return super.andReturn(value); - } - - } - - public class DeleteVersionLockCommandExpectation extends GenericYumCommandExpectation { - - private DeleteVersionLockCommandExpectation(String yumPackage) { - super(CommandType.deleteVersionLock, yumPackage); - } - - } - - public class QueryInstalledExpectation { - private final String packageName; - - public QueryInstalledExpectation(String packageName) { - this.packageName = packageName; - } - - /** Package name to return or null if package is not installed */ - public YumTester andReturn(YumPackageName yumPackage) { - TestChildProcess2 process = new TestChildProcess2( - yumPackage == null ? 1 : 0, - yumPackage == null ? "not installed" : String.join("\n", - yumPackage.getName(), - yumPackage.getEpoch().orElse("(none)"), - yumPackage.getVersion().orElseThrow(() -> new IllegalArgumentException("Version must be set")), - yumPackage.getRelease().orElseThrow(() -> new IllegalArgumentException("Release must be set")), - yumPackage.getArchitecture().orElse("(none)"))); - - terminal.expectCommand("rpm -q " + packageName + " --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", process); - return YumTester.this; - } - } - - private enum CommandType { - install("install"), upgrade("upgrade"), remove("remove"), installFixed("install"), deleteVersionLock("versionlock delete"); - - private final String command; - CommandType(String command) { - this.command = command; - } - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/package-info.java deleted file mode 100644 index 89a39151974..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java deleted file mode 100644 index 34d0f555661..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.WireguardKeyWithTimestamp; -import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; - -import java.util.List; - -/** - * A wireguard peer. Sorted by hostname. IP addresses are sorted by version, IPv6 first. - * The public key should always be non-null. - * - * @author gjoranv - */ -public record WireguardPeer(HostName hostname, - List<VersionedIpAddress> ipAddresses, - WireguardKeyWithTimestamp keyWithTimestamp) implements Comparable<WireguardPeer> { - - public WireguardPeer { - if (ipAddresses.isEmpty()) throw new IllegalArgumentException("No IP addresses for peer node " + hostname.value()); - ipAddresses = ipAddresses.stream().sorted().toList(); - } - - @Override - public int compareTo(WireguardPeer o) { - return hostname.value().compareTo(o.hostname.value()); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/package-info.java deleted file mode 100644 index fb5a055915a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/sh/node-admin.sh b/node-admin/src/main/sh/node-admin.sh deleted file mode 100755 index a2ce59e57a8..00000000000 --- a/node-admin/src/main/sh/node-admin.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -findpath () { - myname=${0} - mypath=${myname%/*} - myname=${myname##*/} - empty_if_start_slash=${mypath%%/*} - if [ "${empty_if_start_slash}" ]; then - mypath=$(pwd)/${mypath} - fi - if [ "$mypath" ] && [ -d "$mypath" ]; then - return - fi - mypath=$(pwd) - if [ -f "${mypath}/${myname}" ]; then - return - fi - echo "FATAL: Could not figure out the path where $myname lives from $0" - exit 1 -} - -COMMON_ENV=libexec/vespa/common-env.sh - -source_common_env () { - if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then - export VESPA_HOME - common_env=$VESPA_HOME/$COMMON_ENV - if [ -f "$common_env" ]; then - . $common_env - return - fi - fi - return 1 -} - -findroot () { - source_common_env && return - if [ "$VESPA_HOME" ]; then - echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" - exit 1 - fi - if [ "$ROOT" ] && [ -d "$ROOT" ]; then - VESPA_HOME="$ROOT" - source_common_env && return - fi - findpath - while [ "$mypath" ]; do - VESPA_HOME=${mypath} - source_common_env && return - mypath=${mypath%/*} - done - echo "FATAL: missing VESPA_HOME environment variable" - echo "Could not locate $COMMON_ENV anywhere" - exit 1 -} - -findhost () { - if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 - fi - validate="${VESPA_HOME}/bin/vespa-validate-hostname" - if [ -f "$validate" ]; then - "$validate" "${VESPA_HOSTNAME}" || exit 1 - fi - export VESPA_HOSTNAME -} - -findroot -findhost - -ROOT=${VESPA_HOME%/} -export ROOT - -# END environment bootstrap section - -Usage() { - cat <<EOF -Usage: ${0##*/} [start|stop] -Manage standalone node admin -EOF - - exit 1 -} - -Start() { - "$VESPA_HOME"/libexec/vespa/standalone-container.sh start -s node-admin -u root "$@" -} - -Stop() { - "$VESPA_HOME"/libexec/vespa/standalone-container.sh stop -s node-admin -u root "$@" -} - -if (( $# == 0 )); then - Usage -fi - -command="$1" -shift - -case "$command" in - start) Start "$@" ;; - stop) Stop "$@" ;; - restart) - Stop "$@" - Start "$@" - ;; - *) Usage ;; -esac diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java deleted file mode 100644 index d3982af14e4..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import com.yahoo.vespa.hosted.node.admin.container.ContainerId; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.util.Map; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.sharesToWeight; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.weightToShares; -import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Device; -import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - */ -public class CgroupTest { - - private static final ContainerId containerId = new ContainerId("4aec78cc"); - - private final FileSystem fileSystem = TestFileSystem.create(); - private final Cgroup containerCgroup = Cgroup.root(fileSystem).resolveContainer(containerId); - private final CpuController containerCpu = containerCgroup.cpu(); - private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); - private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories(); - - @Test - public void updates_cpu_quota_and_period() { - assertEquals(Optional.empty(), containerCgroup.cpu().readMax()); - - cgroupRoot.resolve("cpu.max").writeUtf8File("max 100000\n"); - assertEquals(Optional.of(new CpuController.Max(Size.max(), 100000)), containerCpu.readMax()); - - cgroupRoot.resolve("cpu.max").writeUtf8File("456 123456\n"); - assertEquals(Optional.of(new CpuController.Max(Size.from(456), 123456)), containerCpu.readMax()); - - containerCgroup.cpu().updateMax(context, 456, 123456); - - assertTrue(containerCgroup.cpu().updateMax(context, 654, 123456)); - assertEquals(Optional.of(new CpuController.Max(Size.from(654), 123456)), containerCpu.readMax()); - assertEquals("654 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File()); - - assertTrue(containerCgroup.cpu().updateMax(context, -1, 123456)); - assertEquals(Optional.of(new CpuController.Max(Size.max(), 123456)), containerCpu.readMax()); - assertEquals("max 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File()); - } - - @Test - public void updates_cpu_shares() { - assertEquals(Optional.empty(), containerCgroup.cpu().readShares()); - - cgroupRoot.resolve("cpu.weight").writeUtf8File("1\n"); - assertEquals(Optional.of(2), containerCgroup.cpu().readShares()); - - assertFalse(containerCgroup.cpu().updateShares(context, 2)); - - assertTrue(containerCgroup.cpu().updateShares(context, 12345)); - assertEquals(Optional.of(12323), containerCgroup.cpu().readShares()); - } - - @Test - public void reads_cpu_stats() { - cgroupRoot.resolve("cpu.stat").writeUtf8File(""" - usage_usec 17794243 - user_usec 16099205 - system_usec 1695038 - nr_periods 12465 - nr_throttled 25 - throttled_usec 14256 - """); - - assertEquals(Map.of(TOTAL_USAGE_USEC, 17794243L, USER_USAGE_USEC, 16099205L, SYSTEM_USAGE_USEC, 1695038L, - TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), containerCgroup.cpu().readStats()); - } - - @Test - public void reads_memory_metrics() { - cgroupRoot.resolve("memory.current").writeUtf8File("2525093888\n"); - assertEquals(2525093888L, containerCgroup.memory().readCurrent().value()); - - cgroupRoot.resolve("memory.max").writeUtf8File("4322885632\n"); - assertEquals(4322885632L, containerCgroup.memory().readMax().value()); - - cgroupRoot.resolve("memory.stat").writeUtf8File(""" - anon 3481600 - file 69206016 - kernel_stack 73728 - slab 3552304 - percpu 262336 - sock 73728 - shmem 8380416 - file_mapped 1081344 - file_dirty 135168 - slab_reclaimable 1424320 - """); - var stats = containerCgroup.memory().readStat(); - assertEquals(69206016L, stats.file().value()); - assertEquals(3481600L, stats.anon().value()); - assertEquals(3552304L, stats.slab().value()); - assertEquals(73728L, stats.sock().value()); - assertEquals(1424320L, stats.slabReclaimable().value()); - } - - @Test - public void shares_to_weight_and_back_is_stable() { - for (int i = 2; i <= 262144; i++) { - int originalShares = i; // Must be effectively final to use in lambda :( - int roundTripShares = weightToShares(sharesToWeight(i)); - int diff = i - roundTripShares; - assertTrue(diff >= 0 && diff <= 27, // ~26.2 shares / weight - () -> "Original shares: " + originalShares + ", round trip shares: " + roundTripShares + ", diff: " + diff); - } - } - - @Test - void reads_io_max() { - assertEquals(Optional.empty(), containerCgroup.io().readMax()); - - cgroupRoot.resolve("io.max").writeUtf8File(""); - assertEquals(Optional.of(Map.of()), containerCgroup.io().readMax()); - - cgroupRoot.resolve("io.max").writeUtf8File(""" - 253:1 rbps=11 wbps=max riops=22 wiops=33 - 253:0 rbps=max wbps=44 riops=max wiops=55 - """); - assertEquals(Map.of(new Device(253, 1), new Max(Size.from(11), Size.max(), Size.from(22), Size.from(33)), - new Device(253, 0), new Max(Size.max(), Size.from(44), Size.max(), Size.from(55))), - containerCgroup.io().readMax().orElseThrow()); - } - - @Test - void writes_io_max() { - Device device = new Device(253, 0); - Max initial = new Max(Size.max(), Size.from(44), Size.max(), Size.from(55)); - assertTrue(containerCgroup.io().updateMax(context, device, initial)); - assertEquals("253:0 rbps=max wbps=44 riops=max wiops=55\n", cgroupRoot.resolve("io.max").readUtf8File()); - - cgroupRoot.resolve("io.max").writeUtf8File(""" - 253:1 rbps=11 wbps=max riops=22 wiops=33 - 253:0 rbps=max wbps=44 riops=max wiops=55 - """); - assertFalse(containerCgroup.io().updateMax(context, device, initial)); - - cgroupRoot.resolve("io.max").writeUtf8File(""); - assertFalse(containerCgroup.io().updateMax(context, device, Max.UNLIMITED)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java deleted file mode 100644 index cb828394249..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.cgroup; - -import org.junit.jupiter.api.Test; - -import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -class IoControllerTest { - - @Test - void parse_io_max() { - assertEquals(Max.UNLIMITED, Max.fromString("")); - assertEquals(new Max(Size.from(1), Size.max(), Size.max(), Size.max()), Max.fromString("rbps=1 wiops=max")); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java deleted file mode 100644 index 910fd8e670a..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.http.HttpVersion; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.entity.BasicHttpEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicStatusLine; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.SocketTimeoutException; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Basic testing of retry logic. - * - * @author dybis - */ -public class ConfigServerApiImplTest { - - private static final int FAIL_RETURN_CODE = 100000; - private static final int TIMEOUT_RETURN_CODE = 100001; - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class TestPojo { - @JsonProperty("foo") - String foo; - @JsonProperty("error-code") - Integer errorCode; - } - - private final String uri1 = "http://host1:666"; - private final String uri2 = "http://host2:666"; - private final List<URI> configServers = List.of(URI.create(uri1), URI.create(uri2)); - private final StringBuilder mockLog = new StringBuilder(); - - private ConfigServerApiImpl configServerApi; - private int mockReturnCode = 200; - - @BeforeEach - public void initExecutor() throws IOException { - CloseableHttpClient httpMock = mock(CloseableHttpClient.class); - when(httpMock.execute(any())).thenAnswer(invocationOnMock -> { - HttpGet get = (HttpGet) invocationOnMock.getArguments()[0]; - mockLog.append(get.getMethod()).append(" ").append(get.getURI()).append(" "); - - switch (mockReturnCode) { - case FAIL_RETURN_CODE -> throw new RuntimeException("FAIL"); - case TIMEOUT_RETURN_CODE -> throw new SocketTimeoutException("read timed out"); - } - - BasicStatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, mockReturnCode, null); - BasicHttpEntity entity = new BasicHttpEntity(); - String returnMessage = "{\"foo\":\"bar\", \"no\":3, \"error-code\": " + mockReturnCode + "}"; - InputStream stream = new ByteArrayInputStream(returnMessage.getBytes(StandardCharsets.UTF_8)); - entity.setContent(stream); - - CloseableHttpResponse response = mock(CloseableHttpResponse.class); - when(response.getEntity()).thenReturn(entity); - when(response.getStatusLine()).thenReturn(statusLine); - - return response; - }); - configServerApi = ConfigServerApiImpl.createForTestingWithClient(configServers, httpMock); - } - - @Test - void testBasicParsingSingleServer() { - TestPojo answer = configServerApi.get("/path", TestPojo.class); - assertEquals(answer.foo, "bar"); - assertLogStringContainsGETForAHost(); - } - - @Test - void testBasicFailure() { - assertThrows(HttpException.class, () -> { - // Server is returning 400, no retries. - mockReturnCode = 400; - - TestPojo testPojo = configServerApi.get("/path", TestPojo.class); - assertEquals(testPojo.errorCode.intValue(), mockReturnCode); - assertLogStringContainsGETForAHost(); - }); - } - - @Test - void testBasicSuccessWithNoRetries() { - // Server is returning 201, no retries. - mockReturnCode = 201; - - TestPojo testPojo = configServerApi.get("/path", TestPojo.class); - assertEquals(testPojo.errorCode.intValue(), mockReturnCode); - assertLogStringContainsGETForAHost(); - } - - @Test - void testBasicSuccessWithCustomTimeouts() { - mockReturnCode = TIMEOUT_RETURN_CODE; - - var params = new ConfigServerApi.Params<TestPojo>(); - params.setConnectionTimeout(Duration.ofSeconds(3)); - - try { - configServerApi.get("/path", TestPojo.class, params); - fail(); - } catch (ConnectionException e) { - assertNotNull(e.getCause()); - assertEquals("read timed out", e.getCause().getMessage()); - } - } - - @Test - void testRetries() { - // Client is throwing exception, should be retries. - mockReturnCode = FAIL_RETURN_CODE; - try { - configServerApi.get("/path", TestPojo.class); - fail("Expected failure"); - } catch (Exception e) { - // ignore - } - - List<String> log = List.of(mockLog.toString().split(" ")); - assertTrue(log.containsAll(List.of("GET http://host1:666/path", "GET http://host2:666/path"))); - } - - @Test - void testNoRetriesOnBadHttpResponseCode() { - // Client is throwing exception, should be retries. - mockReturnCode = 503; - try { - configServerApi.get("/path", TestPojo.class); - fail("Expected failure"); - } catch (Exception e) { - // ignore - } - - assertLogStringContainsGETForAHost(); - } - - @Test - void testForbidden() { - mockReturnCode = 403; - try { - configServerApi.get("/path", TestPojo.class); - fail("Expected exception"); - } catch (HttpException.ForbiddenException e) { - // ignore - } - assertLogStringContainsGETForAHost(); - } - - @Test - void testNotFound() { - // Server is returning 404, special exception is thrown. - mockReturnCode = 404; - try { - configServerApi.get("/path", TestPojo.class); - fail("Expected exception"); - } catch (HttpException.NotFoundException e) { - // ignore - } - assertLogStringContainsGETForAHost(); - } - - @Test - void testConflict() { - // Server is returning 409, no exception is thrown. - mockReturnCode = 409; - configServerApi.get("/path", TestPojo.class); - assertLogStringContainsGETForAHost(); - } - - private void assertLogStringContainsGETForAHost() { - String logString = mockLog.toString(); - assertTrue((logString.equals("GET http://host1:666/path ") || logString.equals("GET http://host2:666/path ")), - "log does not contain expected entries:" + logString); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java deleted file mode 100644 index 430da856cfa..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.cores; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; -import com.yahoo.test.json.JsonTestHelper; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.node.admin.configserver.StandardConfigServerResponse; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.bindings.ReportCoreDumpRequest; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.time.Instant; -import java.util.List; -import java.util.Optional; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author hakonhall - */ -class CoresTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final ObjectMapper mapper = new ObjectMapper(); - private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); - private final Cores cores = new CoresImpl(configServerApi); - private final HostName hostname = HostName.of("foo.com"); - private final String id = "5c987afb-347a-49ee-a0c5-bef56bbddeb0"; - private final CoreDumpMetadata metadata = new CoreDumpMetadata() - .setType(CoreDumpMetadata.Type.OOM) - .setCreated(Instant.ofEpochMilli(12345678)) - .setKernelVersion("4.18.0-372.26.1.el8_6.x86_64") - .setCpuMicrocodeVersion("0x1000065") - .setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813")) - .setDecryptionToken("987def") - .setDockerImage(DockerImage.fromString("us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8")) - .setBinPath("/usr/bin/java") - .setVespaVersion("8.68.8") - .setBacktraceAllThreads(List.of("Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", - "Debugger attached successfully.", - " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", - " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)")) - .setBacktrace(List.of("Example", "of", "backtrace")); - - @Test - void reportOK() { - var oKResponse = new StandardConfigServerResponse(); - oKResponse.message = "OK"; - when(configServerApi.post(any(), any(), any())).thenReturn(oKResponse); - - cores.report(hostname, id, metadata); - - var pathCaptor = ArgumentCaptor.forClass(String.class); - var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class); - verify(configServerApi, times(1)).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any()); - - assertEquals("/cores/v1/report/" + hostname + "/" + id, pathCaptor.getValue()); - - assertEquals(""" - { - "backtrace": [ - "Example", - "of", - "backtrace" - ], - "backtrace_all_threads": [ - "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", - "Debugger attached successfully.", - " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", - " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)" - ], - "bin_path": "/usr/bin/java", - "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", - "cpu_microcode_version": "0x1000065", - "created": 12345678, - "decryption_token": "987def", - "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", - "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", - "type": "OOM", - "vespa_version": "8.68.8" - }""", - JsonTestHelper.normalize(uncheck(() -> mapper.writeValueAsString(bodyJsonPojoCaptor.getValue())))); - } - - @Test - void reportFails() { - var response = new StandardConfigServerResponse(); - response.errorCode = "503"; - response.message = "error detail"; - when(configServerApi.post(any(), any(), any())).thenReturn(response); - - assertThrows(ConfigServerException.class, - () -> cores.report(hostname, "abcde-1234", metadata), - "Failed to report core dump at Optional[/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813]: error detail 503"); - - var pathCaptor = ArgumentCaptor.forClass(String.class); - var bodyJsonPojoCaptor = ArgumentCaptor.forClass(Object.class); - verify(configServerApi).post(pathCaptor.capture(), bodyJsonPojoCaptor.capture(), any()); - } - - @Test - void serialization() { - Path path = fileSystem.getPath("/foo.json"); - ReportCoreDumpRequest request = new ReportCoreDumpRequest().fillFrom(metadata); - request.save(path); - assertEquals(""" - { - "backtrace": [ - "Example", - "of", - "backtrace" - ], - "backtrace_all_threads": [ - "Attaching to core /opt/vespa/var/crash/processing/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813 from executable /usr/bin/java, please wait...", - "Debugger attached successfully.", - " - com.yahoo.jdisc.core.TimeoutManagerImpl$ManagerTask.run() @bci=3, line=123 (Interpreted frame)", - " - java.lang.Thread.run() @bci=11, line=833 (Interpreted frame)" - ], - "bin_path": "/usr/bin/java", - "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", - "cpu_microcode_version": "0x1000065", - "created": 12345678, - "decryption_token": "987def", - "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", - "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", - "type": "OOM", - "vespa_version": "8.68.8" - }""", - JsonTestHelper.normalize(new UnixPath(path).readUtf8File())); - - Optional<ReportCoreDumpRequest> loaded = ReportCoreDumpRequest.load(path); - assertTrue(loaded.isPresent()); - var meta = new CoreDumpMetadata(); - loaded.get().populateMetadata(meta, fileSystem); - assertEquals(metadata, meta); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java deleted file mode 100644 index 664e25bc744..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.flags; - -import com.yahoo.vespa.flags.FlagId; -import com.yahoo.vespa.flags.json.FlagData; -import com.yahoo.vespa.flags.json.wire.WireFlagData; -import com.yahoo.vespa.flags.json.wire.WireFlagDataList; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author hakonhall - */ -public class RealFlagRepositoryTest { - private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); - private final RealFlagRepository repository = new RealFlagRepository(configServerApi); - - @Test - void test() { - WireFlagDataList list = new WireFlagDataList(); - list.flags = new ArrayList<>(); - list.flags.add(new WireFlagData()); - list.flags.get(0).id = "id1"; - - when(configServerApi.get(any(), eq(WireFlagDataList.class))).thenReturn(list); - Map<FlagId, FlagData> allFlagData = repository.getAllFlagData(); - assertEquals(1, allFlagData.size()); - assertTrue(allFlagData.containsKey(new FlagId("id1"))); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java deleted file mode 100644 index d91e9befab9..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * - * @author smorgrav - */ -public class AclTest { - - private static final Acl aclCommon = new Acl( - Set.of(1234, 453), Set.of(4321), - testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2", "fe80::3"), - Set.of()); - - private static final Acl aclWithoutPorts = new Acl( - Set.of(), Set.of(), - testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2"), - Set.of()); - - @Test - void no_trusted_ports() { - String listRulesIpv4 = String.join("\n", aclWithoutPorts.toRules(IPVersion.IPv4)); - assertEquals( - """ - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -s 192.1.2.2/32 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - listRulesIpv4); - } - - @Test - void ipv4_rules() { - String listRulesIpv4 = String.join("\n", aclCommon.toRules(IPVersion.IPv4)); - assertEquals( - """ - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT - -A INPUT -p udp -m multiport --dports 4321 -j ACCEPT - -A INPUT -s 192.1.2.2/32 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - listRulesIpv4); - } - - @Test - void ipv6_rules() { - String listRulesIpv6 = String.join("\n", aclCommon.toRules(IPVersion.IPv6)); - assertEquals( - """ - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p ipv6-icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT - -A INPUT -p udp -m multiport --dports 4321 -j ACCEPT - -A INPUT -s fb00::1/128 -j ACCEPT - -A INPUT -s fe80::2/128 -j ACCEPT - -A INPUT -s fe80::3/128 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", listRulesIpv6); - } - - @Test - void ipv6_rules_stable_order() { - Acl aclCommonDifferentOrder = new Acl( - Set.of(453, 1234), Set.of(4321), - testNodes(Set.of(), "fe80::2", "192.1.2.2", "fb00::1", "fe80::3"), - Set.of()); - - for (IPVersion ipVersion : IPVersion.values()) { - assertEquals(aclCommon.toRules(ipVersion), aclCommonDifferentOrder.toRules(ipVersion)); - } - } - - @Test - void trusted_networks() { - Acl acl = new Acl(Set.of(4080), Set.of(), testNodes(Set.of(), "127.0.0.1"), Set.of("10.0.0.0/24", "2001:db8::/32")); - - assertEquals(""" - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT - -A INPUT -s 127.0.0.1/32 -j ACCEPT - -A INPUT -s 10.0.0.0/24 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - String.join("\n", acl.toRules(IPVersion.IPv4))); - - assertEquals(""" - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p ipv6-icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT - -A INPUT -s 2001:db8::/32 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", - String.join("\n", acl.toRules(IPVersion.IPv6))); - } - - @Test - void config_server_acl() { - Set<Acl.Node> testNodes = Stream.concat(testNodes(NodeType.config, Set.of(), "172.17.0.41", "172.17.0.42", "172.17.0.43").stream(), - testNodes(NodeType.tenant, Set.of(19070), "172.17.0.81", "172.17.0.82", "172.17.0.83").stream()) - .collect(Collectors.toSet()); - Acl acl = new Acl(Set.of(22, 4443), Set.of(), testNodes, Set.of()); - assertEquals(""" - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 172.17.0.41/32 -j ACCEPT - -A INPUT -s 172.17.0.42/32 -j ACCEPT - -A INPUT -s 172.17.0.43/32 -j ACCEPT - -A INPUT -s 172.17.0.81/32 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -s 172.17.0.82/32 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -s 172.17.0.83/32 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - String.join("\n", acl.toRules(IPVersion.IPv4))); - - Set<Acl.Node> testNodes2 = Stream.concat(testNodes(NodeType.config, Set.of(), "2001:db8::41", "2001:db8::42", "2001:db8::43").stream(), - testNodes(NodeType.tenant, Set.of(19070), "2001:db8::81", "2001:db8::82", "2001:db8::83").stream()) - .collect(Collectors.toSet()); - Acl acl2 = new Acl(Set.of(22, 4443), Set.of(), testNodes2, Set.of()); - - assertEquals(""" - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p ipv6-icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 2001:db8::41/128 -j ACCEPT - -A INPUT -s 2001:db8::42/128 -j ACCEPT - -A INPUT -s 2001:db8::43/128 -j ACCEPT - -A INPUT -s 2001:db8::81/128 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -s 2001:db8::82/128 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -s 2001:db8::83/128 -p tcp -m multiport --dports 19070 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", - String.join("\n", acl2.toRules(IPVersion.IPv6))); - } - - private static Set<Acl.Node> testNodes(Set<Integer> ports, String... address) { - return testNodes(NodeType.tenant, ports, address); - } - - private static Set<Acl.Node> testNodes(NodeType nodeType, Set<Integer> ports, String... address) { - return Arrays.stream(address) - .map(addr -> new Acl.Node("hostname", addr, ports)) - .collect(Collectors.toUnmodifiableSet()); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java deleted file mode 100644 index b236c223078..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.vespa.hosted.provision.Node; -import org.junit.jupiter.api.Test; - -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class NodeStateTest { - - @Test - void is_equal_to_node_repository_states() { - Set<String> nodeRepositoryStates = Stream.of(Node.State.values()).map(Enum::name).collect(Collectors.toSet()); - Set<String> nodeAdminStates = Stream.of(NodeState.values()).map(Enum::name).collect(Collectors.toSet()); - - assertEquals(nodeAdminStates, nodeRepositoryStates); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java deleted file mode 100644 index 4100b3cf102..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; - -import com.yahoo.application.Networking; -import com.yahoo.application.container.JDisc; -import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.config.provision.WireguardKeyWithTimestamp; -import com.yahoo.config.provision.host.FlavorOverrides; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; -import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler; -import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.ServerSocket; -import java.net.URI; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * Tests the NodeRepository class used for talking to the node repository. It uses a mock from the node repository - * which already contains some data. - * - * @author dybdahl - */ -public class RealNodeRepositoryTest { - - private static final double delta = 0.00000001; - private JDisc container; - private NodeRepository nodeRepositoryApi; - - private int findRandomOpenPort() throws IOException { - try (ServerSocket socket = new ServerSocket(0)) { - socket.setReuseAddress(true); - return socket.getLocalPort(); - } - } - - /** - * Starts NodeRepository with - * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors} - * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository} - * {@link NodesV2ApiHandler} - * These classes define some test data that is used in these tests. - */ - @BeforeEach - public void startContainer() throws Exception { - Exception lastException = null; - - // This tries to bind a random open port for the node-repo mock, which is a race condition, so try - // a few times before giving up - for (int i = 0; i < 3; i++) { - try { - int port = findRandomOpenPort(); - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, SystemName.main, CloudAccount.from("123456789012")), Networking.enable); - ConfigServerApi configServerApi = ConfigServerApiImpl.createForTesting( - List.of(URI.create("http://127.0.0.1:" + port))); - waitForJdiscContainerToServe(configServerApi); - return; - } catch (RuntimeException e) { - lastException = e; - } - } - throw new RuntimeException("Failed to bind a port in three attempts, giving up", lastException); - } - - private void waitForJdiscContainerToServe(ConfigServerApi configServerApi) throws InterruptedException { - Instant start = Instant.now(); - nodeRepositoryApi = new RealNodeRepository(configServerApi); - while (Instant.now().minusSeconds(120).isBefore(start)) { - try { - nodeRepositoryApi.getNodes("foobar"); - return; - } catch (Exception e) { - Thread.sleep(100); - } - } - throw new RuntimeException("Could not get answer from container."); - } - - @AfterEach - public void stopContainer() { - if (container != null) { - container.close(); - } - } - - @Test - void testGetContainersToRunApi() { - String dockerHostHostname = "dockerhost1.yahoo.com"; - - List<NodeSpec> containersToRun = nodeRepositoryApi.getNodes(dockerHostHostname); - assertEquals(1, containersToRun.size()); - NodeSpec node = containersToRun.get(0); - assertEquals("host4.yahoo.com", node.hostname()); - assertEquals(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"), node.wantedDockerImage().get()); - assertEquals(NodeState.active, node.state()); - assertEquals(Long.valueOf(0), node.wantedRestartGeneration().get()); - assertEquals(Long.valueOf(0), node.currentRestartGeneration().get()); - assertEquals(1, node.vcpu(), delta); - assertEquals(4, node.memoryGb(), delta); - assertEquals(100, node.diskGb(), delta); - } - - @Test - void testGetContainer() { - String hostname = "host4.yahoo.com"; - Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname); - assertTrue(node.isPresent()); - assertEquals(hostname, node.get().hostname()); - assertEquals(CloudAccount.from("123456789012"), node.get().cloudAccount()); - } - - @Test - void testGetContainerForNonExistingNode() { - String hostname = "host-that-does-not-exist"; - Optional<NodeSpec> node = nodeRepositoryApi.getOptionalNode(hostname); - assertFalse(node.isPresent()); - } - - @Test - void testUpdateNodeAttributes() { - var hostname = "host4.yahoo.com"; - var dockerImage = "registry.example.com/repo/image-1:6.2.3"; - var wireguardKey = WireguardKey.from("111122223333444455556666777788889999000042c="); - var wireguardKeyTimestamp = Instant.ofEpochMilli(123L); // Instant from clock in MockNodeRepository - var keyWithTimestamp = new WireguardKeyWithTimestamp(wireguardKey, wireguardKeyTimestamp); - - nodeRepositoryApi.updateNodeAttributes( - hostname, - new NodeAttributes() - .withRestartGeneration(1) - .withDockerImage(DockerImage.fromString(dockerImage)) - .withWireguardPubkey(wireguardKey)); - - NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode(hostname).orElseThrow(); - assertEquals(1, hostSpec.currentRestartGeneration().orElseThrow()); - assertEquals(dockerImage, hostSpec.currentDockerImage().orElseThrow().asString()); - assertEquals(keyWithTimestamp, hostSpec.wireguardKeyWithTimestamp().orElseThrow()); - } - - @Test - void testMarkAsReady() { - nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.dirty); - nodeRepositoryApi.setNodeState("host5.yahoo.com", NodeState.ready); - - try { - nodeRepositoryApi.setNodeState("host4.yahoo.com", NodeState.ready); - fail("Should not be allowed to be marked ready as it is not registered as provisioned, dirty, failed or parked"); - } catch (RuntimeException ignored) { - // expected - } - - try { - nodeRepositoryApi.setNodeState("host101.yahoo.com", NodeState.ready); - fail("Expected failure because host101 does not exist"); - } catch (RuntimeException ignored) { - // expected - } - } - - @Test - void testAddNodes() { - AddNode host = AddNode.forHost("host123.domain.tld", - "id1", - "default", - Optional.of(FlavorOverrides.ofDisk(123)), - NodeType.confighost, - Set.of("::1"), Set.of("::2", "::3")); - - NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow, NodeResources.StorageType.local); - AddNode node = AddNode.forNode("host123-1.domain.tld", "id1", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); - - assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent()); - nodeRepositoryApi.addNodes(List.of(host, node)); - - NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow(); - assertEquals("id1", hostSpec.id()); - assertEquals("default", hostSpec.flavor()); - assertEquals(123, hostSpec.diskGb(), 0); - assertEquals(NodeType.confighost, hostSpec.type()); - assertEquals(NodeResources.Architecture.x86_64, hostSpec.resources().architecture()); - - NodeSpec nodeSpec = nodeRepositoryApi.getOptionalNode("host123-1.domain.tld").orElseThrow(); - assertEquals(nodeResources, nodeSpec.resources()); - assertEquals(NodeType.config, nodeSpec.type()); - } - - @Test - void wireguard_peer_config_can_be_retrieved_for_configservers_and_exclave_nodes() { - - //// Configservers //// - - List<WireguardPeer> cfgPeers = nodeRepositoryApi.getConfigserverPeers(); - - // cfg2 does not have a wg public key, so should not be included - assertEquals(1, cfgPeers.size()); - - assertWireguardPeer(cfgPeers.get(0), "cfg1.yahoo.com", - "::201:1", - "lololololololololololololololololololololoo=", - 456L); - - //// Exclave nodes //// - - List<WireguardPeer> exclavePeers = nodeRepositoryApi.getExclavePeers(); - - // host3 does not have a wg public key, so should not be included - assertEquals(1, exclavePeers.size()); - - assertWireguardPeer(exclavePeers.get(0), "dockerhost2.yahoo.com", - "::101:1", - "000011112222333344445555666677778888999900c=", - 123L); - } - - private void assertWireguardPeer(WireguardPeer peer, String hostname, String ipv6, - String publicKey, long keyTimestamp) { - assertEquals(hostname, peer.hostname().value()); - assertEquals(1, peer.ipAddresses().size()); - assertIp(peer.ipAddresses().get(0), ipv6, 6); - var expectedKeyWithTimestamp = new WireguardKeyWithTimestamp(WireguardKey.from(publicKey), - Instant.ofEpochMilli(keyTimestamp)); - assertEquals(expectedKeyWithTimestamp, peer.keyWithTimestamp()); - } - - private void assertIp(VersionedIpAddress ip, String expectedIp, int expectedVersion) { - assertEquals(expectedIp, ip.asString()); - assertEquals(expectedVersion, ip.version().version()); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java deleted file mode 100644 index 8de5986739e..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNodeTest.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.yahoo.test.json.JsonTestHelper; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author hakonhall - */ -public class NodeRepositoryNodeTest { - private static final ObjectMapper mapper = new ObjectMapper(); - private final NodeRepositoryNode node = new NodeRepositoryNode(); - private final NodeAttributes attributes = new NodeAttributes(); - - - /** - * Test both how NodeRepositoryNode serialize, and the serialization of an empty NodeRepositoryNode - * patched with a NodeAttributes, as they work in tandem: - * NodeAttributes -> NodeRepositoryNode -> JSON. - */ - @Test - void testReportsSerialization() { - // Make sure we don't accidentally patch with "reports": null, as that actually means removing all reports. - assertEquals(JsonInclude.Include.NON_NULL, NodeRepositoryNode.class.getAnnotation(JsonInclude.class).value()); - - // Absent report and unmodified attributes => nothing about reports in JSON - node.reports = null; - assertNodeAndAttributes("{}"); - - // Make sure we're able to patch with a null report value ("reportId": null), as that means removing the report. - node.reports = new HashMap<>(); - node.reports.put("rid", null); - attributes.withReportRemoved("rid"); - assertNodeAndAttributes("{\"reports\": {\"rid\": null}}"); - - // Add ridTwo report to node - ObjectNode reportJson = mapper.createObjectNode(); - reportJson.set(BaseReport.CREATED_FIELD, mapper.valueToTree(3)); - reportJson.set(BaseReport.DESCRIPTION_FIELD, mapper.valueToTree("desc")); - node.reports.put("ridTwo", reportJson); - - // Add ridTwo report to attributes - BaseReport reportTwo = new BaseReport(3L, "desc", null); - attributes.withReport("ridTwo", reportTwo.toJsonNode()); - - // Verify node serializes to expected, as well as attributes patched on node. - assertNodeAndAttributes("{\"reports\": {\"rid\": null, \"ridTwo\": {\"createdMillis\": 3, \"description\": \"desc\"}}}"); - } - - private void assertNodeAndAttributes(String expectedJson) { - assertNodeJson(node, expectedJson); - assertNodeJson(RealNodeRepository.nodeRepositoryNodeFromNodeAttributes(attributes), expectedJson); - } - - private void assertNodeJson(NodeRepositoryNode node, String json) { - JsonNode expected = uncheck(() -> mapper.readTree(json)); - JsonNode actual = uncheck(() -> mapper.valueToTree(node)); - JsonTestHelper.assertJsonEquals(actual, expected); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java deleted file mode 100644 index 69e79ec8720..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReportTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; - -import com.yahoo.test.json.JsonTestHelper; -import org.junit.jupiter.api.Test; - -import static com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport.Type.SOFT_FAIL; -import static com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport.Type.UNSPECIFIED; -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author hakonhall - */ -public class BaseReportTest { - private static final String JSON_1 = "{\"createdMillis\": 1, \"description\": \"desc\"}"; - private static final String JSON_2 = "{\"createdMillis\": 1, \"description\": \"desc\", \"type\": \"SOFT_FAIL\"}"; - - @Test - void testSerialization1() { - JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc", SOFT_FAIL).toJsonNode(), - JSON_2); - JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc", SOFT_FAIL).toJsonNode(), - "{\"description\": \"desc\", \"type\": \"SOFT_FAIL\"}"); - JsonTestHelper.assertJsonEquals(new BaseReport(1L, null, SOFT_FAIL).toJsonNode(), - "{\"createdMillis\": 1, \"type\": \"SOFT_FAIL\"}"); - JsonTestHelper.assertJsonEquals(new BaseReport(null, null, SOFT_FAIL).toJsonNode(), - "{\"type\": \"SOFT_FAIL\"}"); - - JsonTestHelper.assertJsonEquals(new BaseReport(1L, "desc", null).toJsonNode(), - JSON_1); - JsonTestHelper.assertJsonEquals(new BaseReport(null, "desc", null).toJsonNode(), - "{\"description\": \"desc\"}"); - JsonTestHelper.assertJsonEquals(new BaseReport(1L, null, null).toJsonNode(), - "{\"createdMillis\": 1}"); - JsonTestHelper.assertJsonEquals(new BaseReport(null, null, null).toJsonNode(), - "{}"); - } - - @Test - void testShouldUpdate() { - BaseReport report = new BaseReport(1L, "desc", SOFT_FAIL); - assertFalse(report.updates(report)); - - // createdMillis is ignored - assertFalse(new BaseReport(1L, "desc", SOFT_FAIL).updates(report)); - assertFalse(new BaseReport(2L, "desc", SOFT_FAIL).updates(report)); - assertFalse(new BaseReport(null, "desc", SOFT_FAIL).updates(report)); - - // description is not ignored - assertTrue(new BaseReport(1L, "desc 2", SOFT_FAIL).updates(report)); - assertTrue(new BaseReport(1L, null, SOFT_FAIL).updates(report)); - - // type is not ignored - assertTrue(new BaseReport(1L, "desc", null).updates(report)); - assertTrue(new BaseReport(1L, "desc", BaseReport.Type.HARD_FAIL).updates(report)); - } - - @Test - void testJsonSerialization() { - BaseReport report = BaseReport.fromJson(JSON_2); - assertEquals(1L, (long) report.getCreatedMillisOrNull()); - assertEquals("desc", report.getDescriptionOrNull()); - assertEquals(SOFT_FAIL, report.getTypeOrNull()); - JsonTestHelper.assertJsonEquals(report.toJson(), JSON_2); - } - - @Test - void testUnspecifiedType() { - BaseReport report = new BaseReport(1L, "desc", null); - assertNull(report.getTypeOrNull()); - assertEquals(UNSPECIFIED, report.getType()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java deleted file mode 100644 index bb9c075ad74..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; - -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; -import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; -import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; -import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; -import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class OrchestratorImplTest { - - private static final String hostName = "host123.yahoo.com"; - - private final ConfigServerApiImpl configServerApi = mock(ConfigServerApiImpl.class); - private final OrchestratorImpl orchestrator = new OrchestratorImpl(configServerApi); - - @Test - void testSuspendCall() { - when(configServerApi.put( - eq(OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"), - eq(Optional.empty()), - eq(UpdateHostResponse.class), - any() - )).thenReturn(new UpdateHostResponse(hostName, null)); - - orchestrator.suspend(hostName); - } - - @Test - void testSuspendCallWithFailureReason() { - assertThrows(OrchestratorException.class, () -> { - when(configServerApi.put( - eq(OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended"), - eq(Optional.empty()), - eq(UpdateHostResponse.class), - any() - )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); - - orchestrator.suspend(hostName); - }); - } - - @Test - void testSuspendCallWithNotFound() { - assertThrows(OrchestratorNotFoundException.class, () -> { - when(configServerApi.put(any(String.class), any(), any(), any())) - .thenThrow(new HttpException.NotFoundException("Not Found")); - - orchestrator.suspend(hostName); - }); - } - - @Test - void testSuspendCallWithSomeOtherException() { - assertThrows(RuntimeException.class, () -> { - when(configServerApi.put(any(String.class), any(), any(), any())) - .thenThrow(new RuntimeException("Some parameter was wrong")); - - orchestrator.suspend(hostName); - }); - } - - - @Test - void testResumeCall() { - when(configServerApi.delete( - OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended", - UpdateHostResponse.class - )).thenReturn(new UpdateHostResponse(hostName, null)); - - orchestrator.resume(hostName); - } - - @Test - void testResumeCallWithFailureReason() { - assertThrows(OrchestratorException.class, () -> { - when(configServerApi.delete( - OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended", - UpdateHostResponse.class - )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); - - orchestrator.resume(hostName); - }); - } - - @Test - void testResumeCallWithNotFound() { - assertThrows(OrchestratorNotFoundException.class, () -> { - when(configServerApi.delete( - any(String.class), - any() - )).thenThrow(new HttpException.NotFoundException("Not Found")); - - orchestrator.resume(hostName); - }); - } - - @Test - void testResumeCallWithSomeOtherException() { - assertThrows(RuntimeException.class, () -> { - when(configServerApi.put(any(String.class), any(), any(), any())) - .thenThrow(new RuntimeException("Some parameter was wrong")); - - orchestrator.suspend(hostName); - }); - } - - @Test - void testBatchSuspendCall() { - String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); - - when(configServerApi.put( - eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), - eq(Optional.empty()), - eq(BatchOperationResult.class), - any() - )).thenReturn(BatchOperationResult.successResult()); - - orchestrator.suspend(parentHostName, hostNames); - } - - @Test - void testBatchSuspendCallWithFailureReason() { - assertThrows(OrchestratorException.class, () -> { - String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); - String failureReason = "Failed to suspend"; - - when(configServerApi.put( - eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), - eq(Optional.empty()), - eq(BatchOperationResult.class), - any() - )).thenReturn(new BatchOperationResult(failureReason)); - - orchestrator.suspend(parentHostName, hostNames); - }); - } - - @Test - void testBatchSuspendCallWithSomeException() { - assertThrows(RuntimeException.class, () -> { - String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); - String exceptionMessage = "Exception: Something crashed!"; - - when(configServerApi.put( - eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), - eq(Optional.empty()), - eq(BatchOperationResult.class), - any() - )).thenThrow(new RuntimeException(exceptionMessage)); - - orchestrator.suspend(parentHostName, hostNames); - }); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java deleted file mode 100644 index 478e89cde34..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.state; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class HealthResponseTest { - @Test - void deserializationOfNormalResponse() throws Exception { - String jsonResponse = "{\n" + - " \"metrics\": {\n" + - " \"snapshot\": {\n" + - " \"from\": 1.523614569023E9,\n" + - " \"to\": 1.523614629023E9\n" + - " },\n" + - " \"values\": [\n" + - " {\n" + - " \"name\": \"requestsPerSecond\",\n" + - " \"values\": {\n" + - " \"count\": 121,\n" + - " \"rate\": 2.0166666666666666\n" + - " }\n" + - " },\n" + - " {\n" + - " \"name\": \"latencySeconds\",\n" + - " \"values\": {\n" + - " \"average\": 5.537190082644628E-4,\n" + - " \"count\": 121,\n" + - " \"last\": 0.001,\n" + - " \"max\": 0.001,\n" + - " \"min\": 0,\n" + - " \"rate\": 2.0166666666666666\n" + - " }\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"status\": {\"code\": \"up\"},\n" + - " \"time\": 1523614629451\n" + - "}"; - - HealthResponse response = deserialize(jsonResponse); - - assertEquals(response.status.code, "up"); - } - - private static HealthResponse deserialize(String json) throws Exception { - ObjectMapper mapper = new ObjectMapper(); - - return mapper.readValue(json, HealthResponse.class); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java deleted file mode 100644 index 733a105f047..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver.state; - -import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; -import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; -import org.junit.jupiter.api.Test; - -import java.net.ConnectException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class StateImplTest { - private final ConfigServerApi api = mock(ConfigServerApi.class); - private final StateImpl state = new StateImpl(api); - - @Test - void testWhenUp() { - HealthResponse response = new HealthResponse(); - response.status.code = "up"; - when(api.get(any(), any())).thenReturn(response); - - HealthCode code = state.getHealth(); - assertEquals(HealthCode.UP, code); - } - - @Test - void connectException() { - RuntimeException exception = - ConnectionException.handleException("Error: ", new ConnectException("connection refused")); - when(api.get(any(), any())).thenThrow(exception); - - HealthCode code = state.getHealth(); - assertEquals(HealthCode.DOWN, code); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java deleted file mode 100644 index 45b74368fd8..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.container.image.Image; -import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; - -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; - -/** - * @author mpolden - */ -public class ContainerEngineMock implements ContainerEngine { - - private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>(); - private final Map<String, ImageDownload> images = new ConcurrentHashMap<>(); - private boolean asyncImageDownload = false; - - private final TestTerminal terminal; - - public ContainerEngineMock() { - this(null); - } - - public ContainerEngineMock(TestTerminal terminal) { - this.terminal = terminal; - } - - public ContainerEngineMock asyncImageDownload(boolean enabled) { - this.asyncImageDownload = enabled; - return this; - } - - public ContainerEngineMock completeDownloadOf(DockerImage image) { - String imageId = image.asString(); - ImageDownload download; - while ((download = images.get(imageId)) == null); - download.complete(); - return this; - } - - public ContainerEngineMock setImages(List<Image> images) { - this.images.clear(); - for (var image : images) { - ImageDownload imageDownload = new ImageDownload(image); - imageDownload.complete(); - this.images.put(image.id(), imageDownload); - } - return this; - } - - public ContainerEngineMock addContainers(List<Container> containers) { - for (var container : containers) { - if (this.containers.containsKey(container.name())) { - throw new IllegalArgumentException("Container " + container.name() + " already exists"); - } - this.containers.put(container.name(), container); - } - return this; - } - - public ContainerEngineMock addContainer(Container container) { - return addContainers(List.of(container)); - } - - @Override - public ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources) { - addContainer(createContainer(context, PartialContainer.State.created, containerResources)); - return new ContainerData() { - @Override - public void addFile(ContainerPath path, String data) { - throw new UnsupportedOperationException("addFile not implemented"); - } - - @Override - public void addFile(ContainerPath path, String data, String permissions) { - throw new UnsupportedOperationException("addFile not implemented"); - } - - @Override - public void addDirectory(ContainerPath path, String... permissions) { - throw new UnsupportedOperationException("addDirectory not implemented"); - } - - @Override - public void addSymlink(ContainerPath symlink, Path target) { - throw new UnsupportedOperationException("addSymlink not implemented"); - } - - @Override - public void converge(NodeAgentContext context) { - throw new UnsupportedOperationException("converge not implemented"); - } - }; - } - - @Override - public void startContainer(NodeAgentContext context) { - Container container = requireContainer(context.containerName(), PartialContainer.State.created); - Container newContainer = createContainer(context, PartialContainer.State.running, container.resources()); - containers.put(newContainer.name(), newContainer); - } - - @Override - public void removeContainer(TaskContext context, PartialContainer container) { - requireContainer(container.name()); - containers.remove(container.name()); - } - - @Override - public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { - Container container = requireContainer(context.containerName()); - containers.put(container.name(), new Container(containerId, container.name(), container.createdAt(), container.state(), - container.imageId(), container.image(), - container.labels(), container.pid(), - container.conmonPid(), container.hostname(), - containerResources, container.networks(), - container.managed())); - } - - @Override - public Optional<Container> getContainer(NodeAgentContext context) { - return Optional.ofNullable(containers.get(context.containerName())); - } - - @Override - public List<PartialContainer> listContainers(TaskContext context) { - return List.copyOf(containers.values()); - } - - @Override - public String networkInterface(NodeAgentContext context) { - return "eth0"; - } - - @Override - public CommandResult execute(NodeAgentContext context, UnixUser user, Duration timeout, String... command) { - if (terminal == null) { - return new CommandResult(null, 0, ""); - } - return terminal.newCommandLine(context) - .add(command) - .executeSilently(); - } - - @Override - public CommandResult executeInNetworkNamespace(NodeAgentContext context, CommandLine.Options options, String... command) { - if (terminal == null) { - return new CommandResult(null, 0, ""); - } - return terminal.newCommandLine(context).add(command).execute(options); - } - - @Override - public void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials) { - String imageId = image.asString(); - ImageDownload imageDownload = images.computeIfAbsent(imageId, (ignored) -> new ImageDownload(new Image(imageId, List.of(imageId)))); - if (!asyncImageDownload) { - imageDownload.complete(); - } - imageDownload.awaitCompletion(); - } - - @Override - public boolean hasImage(TaskContext context, DockerImage image) { - ImageDownload download = images.get(image.asString()); - return download != null && download.isComplete(); - } - - @Override - public void removeImage(TaskContext context, String id) { - images.remove(id); - } - - @Override - public List<Image> listImages(TaskContext context) { - return images.values().stream() - .filter(ImageDownload::isComplete) - .map(ImageDownload::image) - .toList(); - } - - private Container requireContainer(ContainerName name) { - return requireContainer(name, null); - } - - private Container requireContainer(ContainerName name, PartialContainer.State wantedState) { - Container container = containers.get(name); - if (container == null) throw new IllegalArgumentException("No such container: " + name); - if (wantedState != null && container.state() != wantedState) throw new IllegalArgumentException("Container is " + container.state() + ", wanted " + wantedState); - return container; - } - - public Container createContainer(NodeAgentContext context, PartialContainer.State state, ContainerResources containerResources) { - return new Container(new ContainerId("id-of-" + context.containerName()), - context.containerName(), - Instant.EPOCH, - state, - "image-id", - context.node().wantedDockerImage().get(), - Map.of(), - 41, - 42, - context.hostname().value(), - containerResources, - List.of(), - true); - } - - private static class ImageDownload { - - private final Image image; - private final CountDownLatch done = new CountDownLatch(1); - - ImageDownload(Image image) { - this.image = Objects.requireNonNull(image); - } - - Image image() { - return image; - } - - boolean isComplete() { - return done.getCount() == 0; - } - - void complete() { - done.countDown(); - } - - void awaitCompletion() { - try { - done.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java deleted file mode 100644 index f9f7a18597c..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNameTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author freva - */ -public class ContainerNameTest { - @Test - void testAlphanumericalContainerName() { - String name = "container123"; - ContainerName containerName = new ContainerName(name); - assertEquals(containerName.asString(), name); - } - - @Test - void testAlphanumericalWithDashContainerName() { - String name = "container-123"; - ContainerName containerName = new ContainerName(name); - assertEquals(containerName.asString(), name); - } - - @Test - void testContainerNameFromHostname() { - assertEquals(new ContainerName("container-123"), ContainerName.fromHostname("container-123.sub.domain.tld")); - } - - @Test - void testAlphanumericalWithSlashContainerName() { - assertThrows(IllegalArgumentException.class, () -> { - new ContainerName("container/123"); - }); - } - - @Test - void testEmptyContainerName() { - assertThrows(IllegalArgumentException.class, () -> { - new ContainerName(""); - }); - } - - @Test - void testNullContainerName() { - assertThrows(NullPointerException.class, () -> { - new ContainerName(null); - }); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java deleted file mode 100644 index a72e926a471..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * @author mpolden - */ -public class ContainerOperationsTest { - - private final TestTaskContext context = new TestTaskContext(); - private final ContainerEngineMock containerEngine = new ContainerEngineMock(); - private final FileSystem fileSystem = TestFileSystem.create(); - private final TestTimer timer = new TestTimer(); - private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer); - - @Test - void no_managed_containers_running() { - Container c1 = createContainer("c1", true); - Container c2 = createContainer("c2", false); - - containerEngine.addContainer(c1); - assertFalse(containerOperations.noManagedContainersRunning(context)); - - containerEngine.removeContainer(context, c1); - assertTrue(containerOperations.noManagedContainersRunning(context)); - - containerEngine.addContainer(c2); - assertTrue(containerOperations.noManagedContainersRunning(context)); - } - - @Test - void retain_managed_containers() { - Container c1 = createContainer("c1", true); - Container c2 = createContainer("c2", true); - Container c3 = createContainer("c3", false); - containerEngine.addContainers(List.of(c1, c2, c3)); - - assertEquals(3, containerEngine.listContainers(context).size()); - containerOperations.retainManagedContainers(context, Set.of(c1.name())); - - assertEquals(List.of(c1.name(), c3.name()), containerEngine.listContainers(context).stream() - .map(PartialContainer::name) - .sorted() - .toList()); - } - - private Container createContainer(String name, boolean managed) { - return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, PartialContainer.State.running, - "image-id", DockerImage.EMPTY, Map.of(), 42, 43, name, - ContainerResources.UNLIMITED, List.of(), managed); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java deleted file mode 100644 index cbc803b6105..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerResourcesTest.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author freva - */ -public class ContainerResourcesTest { - - @Test - void verify_unlimited() { - assertEquals(-1, ContainerResources.UNLIMITED.cpuQuota()); - assertEquals(100_000, ContainerResources.UNLIMITED.cpuPeriod()); - assertEquals(0, ContainerResources.UNLIMITED.cpuShares()); - } - - @Test - void validate_shares() { - new ContainerResources(0, 0, 0); - new ContainerResources(0, 2, 0); - new ContainerResources(0, 2048, 0); - new ContainerResources(0, 262_144, 0); - - assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, -1, 0)); // Negative shares not allowed - assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, 1, 0)); // 1 share not allowed - assertThrows(IllegalArgumentException.class, () -> new ContainerResources(0, 262_145, 0)); - } - - @Test - void cpu_shares_scaling() { - ContainerResources resources = ContainerResources.from(5.3, 2.5, 0); - assertEquals(530_000, resources.cpuQuota()); - assertEquals(100_000, resources.cpuPeriod()); - assertEquals(80, resources.cpuShares()); - } - - private static void assertThrows(Class<? extends Throwable> clazz, Runnable runnable) { - try { - runnable.run(); - fail("Expected " + clazz); - } catch (Throwable e) { - if (!clazz.isInstance(e)) throw e; - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java deleted file mode 100644 index 8cd3d6529c5..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; -import com.yahoo.vespa.hosted.node.admin.cgroup.MemoryController; -import com.yahoo.vespa.hosted.node.admin.cgroup.Size; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.mockito.Answers; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author mpolden - */ -public class ContainerStatsCollectorTest { - - private final TestTerminal testTerminal = new TestTerminal(); - private final ContainerEngineMock containerEngine = new ContainerEngineMock(testTerminal); - private final FileSystem fileSystem = TestFileSystem.create(); - private final Cgroup cgroup = mock(Cgroup.class, Answers.RETURNS_DEEP_STUBS); - private final NodeAgentContext context = NodeAgentContextImpl.builder(NodeSpec.Builder.testSpec("c1").build()) - .fileSystem(TestFileSystem.create()) - .build(); - @Test - void collect() throws Exception { - ContainerStatsCollector collector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem, 24); - ContainerId containerId = new ContainerId("id1"); - int containerPid = 42; - assertTrue(collector.collect(context, containerId, containerPid, "eth0").isEmpty(), "No stats found"); - - mockMemoryStats(containerId); - mockCpuStats(containerId); - mockNetworkStats(containerPid); - - Optional<ContainerStats> stats = collector.collect(context, containerId, containerPid, "eth0"); - assertTrue(stats.isPresent()); - assertEquals(new ContainerStats.CpuStats(24, 6049374780000L, 691675615472L, - 262190000000L, 3L, 1L, 2L), - stats.get().cpuStats()); - assertEquals(new ContainerStats.MemoryStats(470790144L, 1228017664L, 2147483648L), - stats.get().memoryStats()); - assertEquals(Map.of("eth0", new ContainerStats.NetworkStats(22280813L, 4L, 3L, - 19859383L, 6L, 5L)), - stats.get().networks()); - assertEquals(List.of(), stats.get().gpuStats()); - - mockGpuStats(); - stats = collector.collect(context, containerId, containerPid, "eth0"); - assertTrue(stats.isPresent()); - assertEquals(List.of(new ContainerStats.GpuStats(0, 35, 16106127360L, 6144655360L), - new ContainerStats.GpuStats(1, 67, 32212254720L, 19314769920L)), - stats.get().gpuStats()); - } - - private void mockGpuStats() throws IOException { - Path devPath = fileSystem.getPath("/dev"); - Files.createDirectories(devPath); - Files.createFile(devPath.resolve("nvidia0")); - testTerminal.expectCommand("nvidia-smi --query-gpu=index,utilization.gpu,memory.total,memory.free --format=csv,noheader,nounits 2>&1", 0, - """ - 0, 35, 15360, 9500 - 1, 67, 30720, 12300 - """); - } - - private void mockNetworkStats(int pid) { - UnixPath dev = new UnixPath(fileSystem.getPath("/proc/" + pid + "/net/dev")); - dev.createParents().writeUtf8File("Inter-| Receive | Transmit\n" + - " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + - " lo: 36289258 149700 0 0 0 0 0 0 36289258 149700 0 0 0 0 0 0\n" + - " eth0: 22280813 118083 3 4 0 0 0 0 19859383 115415 5 6 0 0 0 0\n"); - } - - private void mockMemoryStats(ContainerId containerId) { - when(cgroup.resolveContainer(eq(containerId)).memory().readCurrent()).thenReturn(Size.from(1228017664L)); - when(cgroup.resolveContainer(eq(containerId)).memory().readMax()).thenReturn(Size.from(2147483648L)); - when(cgroup.resolveContainer(eq(containerId)).memory().readStat()).thenReturn( - new MemoryController.Stats(Size.from(470790144L), Size.from(0), Size.from(0), Size.from(0), Size.from(0))); - } - - private void mockCpuStats(ContainerId containerId) throws IOException { - UnixPath proc = new UnixPath(fileSystem.getPath("/proc")); - proc.createDirectories(); - - when(cgroup.resolveContainer(eq(containerId)).cpu().readStats()).thenReturn(Map.of( - TOTAL_USAGE_USEC, 691675615472L, SYSTEM_USAGE_USEC, 262190000000L, USER_USAGE_USEC, 40900L, - TOTAL_PERIODS, 1L, THROTTLED_PERIODS, 2L, THROTTLED_TIME_USEC, 3L)); - - proc.resolve("stat").writeUtf8File("cpu 7991366 978222 2346238 565556517 1935450 25514479 615206 0 0 0\n" + - "cpu0 387906 61529 99088 23516506 42258 1063359 29882 0 0 0\n" + - "cpu1 271253 49383 86149 23655234 41703 1061416 31885 0 0 0\n" + - "cpu2 349420 50987 93560 23571695 59437 1051977 24461 0 0 0\n" + - "cpu3 328107 50628 93406 23605135 44378 1048549 30199 0 0 0\n" + - "cpu4 267474 50404 99253 23606041 113094 1038572 26494 0 0 0\n" + - "cpu5 309584 50677 94284 23550372 132616 1033661 29436 0 0 0\n" + - "cpu6 477926 56888 121251 23367023 83121 1074930 28818 0 0 0\n" + - "cpu7 335335 29350 106130 23551107 95606 1066394 26156 0 0 0\n" + - "cpu8 323678 28629 99171 23586501 82183 1064708 25403 0 0 0\n" + - "cpu9 329805 27516 98538 23579458 89235 1061561 25140 0 0 0\n" + - "cpu10 291536 26455 93934 23642345 81282 1049736 25228 0 0 0\n" + - "cpu11 271103 25302 90630 23663641 85711 1048781 24291 0 0 0\n" + - "cpu12 323634 63392 100406 23465340 132684 1089157 28319 0 0 0\n" + - "cpu13 348085 49568 100772 23490388 114190 1079474 20948 0 0 0\n" + - "cpu14 310712 51208 90461 23547980 101601 1071940 26712 0 0 0\n" + - "cpu15 360405 52754 94620 23524878 79851 1062050 26836 0 0 0\n" + - "cpu16 367893 52141 98074 23541314 57500 1058968 25242 0 0 0\n" + - "cpu17 412756 51486 101592 23515056 47653 1044874 27467 0 0 0\n" + - "cpu18 287307 25478 106011 23599505 79848 1089812 23160 0 0 0\n" + - "cpu19 275001 24421 98338 23628694 79675 1084074 22083 0 0 0\n" + - "cpu20 288038 24805 94432 23629908 74735 1078501 21915 0 0 0\n" + - "cpu21 295373 25017 91344 23628585 75282 1071019 22026 0 0 0\n" + - "cpu22 326739 25588 90385 23608217 69186 1068494 21108 0 0 0\n" + - "cpu23 452284 24602 104397 23481583 72612 1052462 21985 0 0 0\n" + - "intr 6645352968 64 0 0 0 1481 0 0 0 1 0 0 0 0 0 0 0 39 0 0 0 0 0 0 37 0 0 0 0 0 0 0 0 4334106 1 6949071 5814662 5415344 6939471 6961483 6358810 5271953 6718644 0 126114 126114 126114 126114 126114 126114 126114 126114 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n" + - "ctxt 2495530303\n" + - "btime 1611928223\n" + - "processes 4839481\n" + - "procs_running 4\n" + - "procs_blocked 0\n" + - "softirq 2202631388 4 20504999 46734 54405637 4330276 0 6951 1664780312 10130 458546345\n"); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java deleted file mode 100644 index 37db6895040..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.image; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author mpolden - */ -public class ContainerImageDownloaderTest { - - @Test - @Timeout(5_000) - void test_download() { - ContainerEngineMock podman = new ContainerEngineMock().asyncImageDownload(true); - ContainerImageDownloader downloader = new ContainerImageDownloader(podman, new TestTimer()); - TaskContext context = new TestTaskContext(); - DockerImage image = DockerImage.fromString("registry.example.com/repo/vespa:7.42"); - - assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download started"); - assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download pending"); - podman.completeDownloadOf(image); - boolean downloadCompleted; - while (!(downloadCompleted = downloader.get(context, image, () -> RegistryCredentials.none))) ; - assertTrue(downloadCompleted, "Download completed"); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java deleted file mode 100644 index 71312125cbc..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.image; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.container.Container; -import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; -import com.yahoo.vespa.hosted.node.admin.container.ContainerId; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; -import org.junit.jupiter.api.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - * @author mpolden - */ -public class ContainerImagePrunerTest { - - private final Tester tester = new Tester(); - - @Test - void noImagesMeansNoUnusedImages() { - tester.withExistingImages() - .expectDeletedImages(); - } - - @Test - void singleImageWithoutContainersIsUnused() { - tester.withExistingImages(image("image-1")) - // Even though nothing is using the image, we will keep it for at least 1h - .expectDeletedImagesAfterMinutes(0) - .expectDeletedImagesAfterMinutes(30) - .expectDeletedImagesAfterMinutes(30, "image-1"); - } - - @Test - void singleImageWithContainerIsUsed() { - tester.withExistingImages(image("image-1")) - .withExistingContainers(container("container-1", "image-1")) - .expectDeletedImages(); - } - - @Test - void multipleUnusedImagesAreIdentified() { - tester.withExistingImages(image("image-1"), image("image-2")) - .expectDeletedImages("image-1", "image-2"); - } - - @Test - void unusedImagesWithMultipleTags() { - tester.withExistingImages(image("image-1", "vespa-6", "vespa-6.28", "vespa:latest")) - .expectDeletedImages("vespa-6", "vespa-6.28", "vespa:latest"); - } - - @Test - void unusedImagesWithMultipleUntagged() { - tester.withExistingImages(image("image1", "<none>:<none>"), - image("image2", "<none>:<none>")) - .expectDeletedImages("image1", "image2"); - } - - @Test - void taggedImageWithNoContainersIsUnused() { - tester.withExistingImages(image("image-1", "vespa-6")) - .expectDeletedImages("vespa-6"); - } - - @Test - void reDownloadingImageIsNotImmediatelyDeleted() { - tester.withExistingImages(image("image")) - .expectDeletedImages("image") // After 1h we delete image - .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted - .expectDeletedImagesAfterMinutes(10) - .expectDeletedImages("image"); // 1h after re-download it is deleted again - } - - @Test - void reDownloadingImageIsNotImmediatelyDeletedWhenDeletingByTag() { - tester.withExistingImages(image("image", "my-tag")) - .expectDeletedImages("my-tag") // After 1h we delete image - .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted - .expectDeletedImagesAfterMinutes(10) - .expectDeletedImages("my-tag"); // 1h after re-download it is deleted again - } - - /** Same scenario as in {@link #multipleUnusedImagesAreIdentified()} */ - @Test - void doesNotDeleteExcludedByIdImages() { - tester.withExistingImages(image("image-1"), image("image-2")) - // Normally, image-1 should also be deleted, but because we exclude image-1 only image-2 is deleted - .expectDeletedImages(List.of("image-1"), "image-2"); - } - - /** Same as in {@link #doesNotDeleteExcludedByIdImages()} but with tags */ - @Test - void doesNotDeleteExcludedByTagImages() { - tester.withExistingImages(image("image-1", "vespa:6.288.16"), image("image-2", "vespa:6.289.94")) - .expectDeletedImages(List.of("vespa:6.288.16"), "vespa:6.289.94"); - } - - @Test - void excludingNotDownloadedImageIsNoop() { - tester.withExistingImages(image("image-1", "vespa:6.288.16"), - image("image-2", "vespa:6.289.94")) - .expectDeletedImages(List.of("vespa:6.300.1"), "vespa:6.288.16", "vespa:6.289.94", "rhel-6"); - } - - private static Image image(String id, String... tags) { - return new Image(id, List.of(tags)); - } - - private static Container container(String name, String imageId) { - return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, - Container.State.running, imageId, DockerImage.EMPTY, Map.of(), - 42, 43, name + ".example.com", ContainerResources.UNLIMITED, - List.of(), true); - } - - private static class Tester { - - private final ContainerEngineMock containerEngine = new ContainerEngineMock(); - private final TaskContext context = new TestTaskContext(); - private final TestTimer timer = new TestTimer(); - private final ContainerImagePruner pruner = new ContainerImagePruner(containerEngine, timer); - private final Map<String, Integer> removalCountByImageId = new HashMap<>(); - - private boolean initialized = false; - - private Tester withExistingImages(Image... images) { - containerEngine.setImages(List.of(images)); - return this; - } - - private Tester withExistingContainers(Container... containers) { - containerEngine.addContainers(List.of(containers)); - return this; - } - - private Tester expectDeletedImages(String... imageIds) { - return expectDeletedImagesAfterMinutes(60, imageIds); - } - - private Tester expectDeletedImages(List<String> excludedRefs, String... imageIds) { - return expectDeletedImagesAfterMinutes(60, excludedRefs, imageIds); - } - - private Tester expectDeletedImagesAfterMinutes(int minutesAfter, String... imageIds) { - return expectDeletedImagesAfterMinutes(minutesAfter, List.of(), imageIds); - } - - private Tester expectDeletedImagesAfterMinutes(int minutesAfter, List<String> excludedRefs, String... imageIds) { - if (!initialized) { - // Run once with a very long expiry to initialize internal state of existing images - pruner.removeUnusedImages(context, List.of(), Duration.ofDays(999)); - initialized = true; - } - - timer.advance(Duration.ofMinutes(minutesAfter)); - - pruner.removeUnusedImages(context, excludedRefs, Duration.ofHours(1).minusSeconds(1)); - - List.of(imageIds) - .forEach(imageId -> { - int newValue = removalCountByImageId.getOrDefault(imageId, 0) + 1; - removalCountByImageId.put(imageId, newValue); - - assertTrue(containerEngine.listImages(context).stream().noneMatch(image -> image.id().equals(imageId)), - "Image " + imageId + " removed"); - }); - return this; - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java deleted file mode 100644 index 8e23c7e54b6..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/metrics/MetricsTest.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container.metrics; - -import org.junit.jupiter.api.Test; - -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics.APPLICATION_HOST; -import static com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics.DimensionType.DEFAULT; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class MetricsTest { - private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); - private final Metrics metrics = new Metrics(); - - @Test - void testDefaultValue() { - metrics.declareCounter("some.name", hostDimension); - - assertEquals(getMetricsForDimension(hostDimension).get("some.name"), 0L); - } - - @Test - void testSimpleIncrementMetric() { - Counter counter = metrics.declareCounter("a_counter.value", hostDimension); - - counter.add(5); - counter.add(8); - - Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); - assertEquals(1, latestMetrics.size(), "Expected only 1 metric value to be set"); - assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8 - } - - @Test - void testSimpleGauge() { - Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); - - gauge.sample(42); - gauge.sample(-342.23); - - Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); - assertEquals(1, latestMetrics.size(), "Expected only 1 metric value to be set"); - assertEquals(latestMetrics.get("test.gauge"), -342.23); - } - - @Test - void testRedeclaringSameGauge() { - Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); - gauge.sample(42); - - // Same as hostDimension, but new instance. - Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); - Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); - newGauge.sample(56); - - assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 56.); - } - - @Test - void testSameMetricNameButDifferentDimensions() { - Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); - gauge.sample(42); - - // Not the same as hostDimension. - Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); - Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); - newGauge.sample(56); - - assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 42.); - assertEquals(getMetricsForDimension(newDimension).get("test.gauge"), 56.); - } - - @Test - void testDeletingMetric() { - metrics.declareGauge("test.gauge", hostDimension); - - Dimensions differentDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); - metrics.declareGauge("test.gauge", differentDimension); - - assertEquals(2, metrics.getMetricsByType(DEFAULT).size()); - metrics.deleteMetricByDimension(APPLICATION_HOST, differentDimension, DEFAULT); - assertEquals(1, metrics.getMetricsByType(DEFAULT).size()); - assertEquals(getMetricsForDimension(hostDimension).size(), 1); - assertEquals(getMetricsForDimension(differentDimension).size(), 0); - } - - private Map<String, Number> getMetricsForDimension(Dimensions dimensions) { - return metrics.getOrCreateApplicationMetrics(APPLICATION_HOST, DEFAULT) - .getOrDefault(dimensions, Map.of()) - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue())); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java deleted file mode 100644 index de41da7329b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -/** - * @author freva - */ -public class ContainerFailTest { - - @Test - void test() { - DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); - try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { - ContainerName containerName = new ContainerName("host1"); - String hostname = "host1.test.yahoo.com"; - NodeSpec nodeSpec = NodeSpec.Builder - .testSpec(hostname) - .wantedDockerImage(dockerImage) - .currentDockerImage(dockerImage) - .build(); - tester.addChildNodeRepositoryNode(nodeSpec); - - NodeAgentContext context = NodeAgentContextImpl.builder(nodeSpec).fileSystem(TestFileSystem.create()).build(); - - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); - - tester.containerOperations.removeContainer(context, tester.containerOperations.getContainer(context).get()); - - tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); - - verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any()); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java deleted file mode 100644 index b4d85a5e974..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ProcMeminfo; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ProcMeminfoReader; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.mockito.InOrder; -import org.mockito.Mockito; - -import java.nio.file.FileSystem; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.logging.Logger; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -/** - * @author musum - */ -// Need to deconstruct nodeAdminStateUpdater -public class ContainerTester implements AutoCloseable { - - private static final Logger log = Logger.getLogger(ContainerTester.class.getName()); - static final HostName HOST_HOSTNAME = HostName.of("host.test.yahoo.com"); - - private final Thread loopThread; - private final Phaser phaser = new Phaser(1); - - private final ContainerEngineMock containerEngine = new ContainerEngineMock(); - private final FileSystem fileSystem = TestFileSystem.create(); - private final TestTimer timer = new TestTimer(); - final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer)); - final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); - final Orchestrator orchestrator = mock(Orchestrator.class); - final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - final InOrder inOrder = Mockito.inOrder(containerOperations, nodeRepository, orchestrator, storageMaintainer); - final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - - final NodeAdminStateUpdater nodeAdminStateUpdater; - final NodeAdminImpl nodeAdmin; - - private volatile NodeAdminStateUpdater.State wantedState = NodeAdminStateUpdater.State.RESUMED; - - - ContainerTester(List<DockerImage> images) { - images.forEach(image -> containerEngine.pullImage(null, image, RegistryCredentials.none)); - when(storageMaintainer.diskUsageFor(any())).thenReturn(Optional.empty()); - - IPAddressesMock ipAddresses = new IPAddressesMock(); - ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1"); - ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::"); - for (int i = 1; i < 4; i++) ipAddresses.addAddress("host" + i + ".test.yahoo.com", "f000::" + i); - - NodeSpec hostSpec = NodeSpec.Builder.testSpec(HOST_HOSTNAME.value()).type(NodeType.host).build(); - nodeRepository.updateNodeSpec(hostSpec); - - Metrics metrics = new Metrics(); - FileSystem fileSystem = TestFileSystem.create(); - ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class); - when(procMeminfoReader.read()).thenReturn(new ProcMeminfo(1, 2)); - - NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> - new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none, - storageMaintainer, flagSource, - List.of(), Optional.empty(), Optional.empty(), timer, Duration.ofSeconds(-1), - VespaServiceDumper.DUMMY_INSTANCE, List.of()) { - @Override public void converge(NodeAgentContext context) { - super.converge(context); - phaser.arriveAndAwaitAdvance(); - } - @Override public void stopForHostSuspension(NodeAgentContext context) { - super.stopForHostSuspension(context); - phaser.arriveAndAwaitAdvance(); - } - @Override public void stopForRemoval(NodeAgentContext context) { - super.stopForRemoval(context); - phaser.arriveAndDeregister(); - } - }; - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, timer, Duration.ofMillis(10), Duration.ZERO, procMeminfoReader); - NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> - NodeAgentContextImpl.builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); - nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, - nodeAdmin, HOST_HOSTNAME); - - loopThread = new Thread(() -> { - nodeAdminStateUpdater.start(); - while ( ! phaser.isTerminated()) { - try { - nodeAdminStateUpdater.converge(wantedState); - } catch (RuntimeException e) { - log.info(e.getMessage()); - } - } - nodeAdminStateUpdater.stop(); - }); - loopThread.start(); - } - - /** Adds a node to node-repository mock that is running on this host */ - void addChildNodeRepositoryNode(NodeSpec nodeSpec) { - if (nodeSpec.wantedDockerImage().isPresent()) { - if (!containerEngine.hasImage(null, nodeSpec.wantedDockerImage().get())) { - throw new IllegalArgumentException("Want to use image " + nodeSpec.wantedDockerImage().get() + - ", but that image does not exist in the container engine"); - } - } - - if (nodeRepository.getOptionalNode(nodeSpec.hostname()).isEmpty()) - phaser.register(); - - nodeRepository.updateNodeSpec(new NodeSpec.Builder(nodeSpec) - .parentHostname(HOST_HOSTNAME.value()) - .build()); - } - - void setWantedState(NodeAdminStateUpdater.State wantedState) { - this.wantedState = wantedState; - } - - <T> T inOrder(T t) { - waitSomeTicks(); - return inOrder.verify(t); - } - - void waitSomeTicks() { - try { - // 3 is enough for everyone! (Well, maybe not for all eternity ...) - for (int i = 0; i < 3; i++) - phaser.awaitAdvanceInterruptibly(phaser.arrive(), 1000, TimeUnit.MILLISECONDS); - } - catch (InterruptedException | TimeoutException e) { - throw new RuntimeException(e); - } - } - - public static NodeAgentContext containerMatcher(ContainerName containerName) { - return argThat((ctx) -> ctx.containerName().equals(containerName)); - } - - @Override - public void close() { - phaser.forceTermination(); - try { - loopThread.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java deleted file mode 100644 index 7e874bcd5a7..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; - -/** - * @author freva - */ -public class MultiContainerTest { - - @Test - void test() { - DockerImage image1 = DockerImage.fromString("registry.example.com/repo/image1"); - DockerImage image2 = DockerImage.fromString("registry.example.com/repo/image2"); - try (ContainerTester tester = new ContainerTester(List.of(image1, image2))) { - addAndWaitForNode(tester, "host1.test.yahoo.com", image1); - NodeSpec nodeSpec2 = addAndWaitForNode(tester, "host2.test.yahoo.com", image2); - - tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(nodeSpec2.hostname(), NodeState.dirty).build()); - - ContainerName host2 = new ContainerName("host2"); - tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(host2), any()); - tester.inOrder(tester.storageMaintainer).archiveNodeStorage( - argThat(context -> context.containerName().equals(host2))); - tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.hostname()), eq(NodeState.ready)); - - addAndWaitForNode(tester, "host3.test.yahoo.com", image1); - } - } - - private NodeAgentContext containerMatcher(ContainerName containerName) { - return argThat((ctx) -> ctx.containerName().equals(containerName)); - } - - private NodeSpec addAndWaitForNode(ContainerTester tester, String hostName, DockerImage dockerImage) { - NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostName).wantedDockerImage(dockerImage).build(); - tester.addChildNodeRepositoryNode(nodeSpec); - - ContainerName containerName = ContainerName.fromHostname(hostName); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); - tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), any()); - - return nodeSpec; - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java deleted file mode 100644 index da14c5aa47b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.AddNode; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NoSuchNodeException; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; - -/** - * Mock with some simple logic - * - * @author dybis - */ -public class NodeRepoMock implements NodeRepository { - - private final Map<String, NodeSpec> nodeSpecByHostname = new ConcurrentHashMap<>(); - private volatile Map<String, Acl> aclByHostname = Map.of(); - - @Override - public void addNodes(List<AddNode> nodes) { } - - @Override - public List<NodeSpec> getNodes(String baseHostName) { - return nodeSpecByHostname.values().stream() - .filter(node -> baseHostName.equals(node.parentHostname().orElse(null))) - .toList(); - } - - @Override - public Optional<NodeSpec> getOptionalNode(String hostName) { - return Optional.ofNullable(nodeSpecByHostname.get(hostName)); - } - - @Override - public Map<String, Acl> getAcls(String hostname) { - return aclByHostname; - } - - @Override - public List<WireguardPeer> getExclavePeers() { - throw new UnsupportedOperationException(); - } - - @Override - public List<WireguardPeer> getConfigserverPeers() { - throw new UnsupportedOperationException(); - } - - @Override - public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) { - updateNodeSpec(new NodeSpec.Builder(getNode(hostName)) - .updateFromNodeAttributes(nodeAttributes) - .build()); - } - - @Override - public void setNodeState(String hostName, NodeState nodeState) { - updateNodeSpec(new NodeSpec.Builder(getNode(hostName)) - .state(nodeState) - .build()); - } - - public void updateNodeSpec(NodeSpec nodeSpec) { - nodeSpecByHostname.put(nodeSpec.hostname(), nodeSpec); - } - - public void updateNodeSpec(String hostname, Function<NodeSpec.Builder, NodeSpec.Builder> mapper) { - nodeSpecByHostname.compute(hostname, (__, nodeSpec) -> { - if (nodeSpec == null) throw new NoSuchNodeException(hostname); - return mapper.apply(new NodeSpec.Builder(nodeSpec)).build(); - }); - } - - public void resetNodeSpecs() { - nodeSpecByHostname.clear(); - } - - public void setAcl(Map<String, Acl> aclByHostname) { - this.aclByHostname = Map.copyOf(aclByHostname); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java deleted file mode 100644 index a1440ba8669..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.HOST_HOSTNAME; -import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; - -/** - * Tests rebooting of Docker host - * - * @author musum - */ -public class RebootTest { - - private final String hostname = "host1.test.yahoo.com"; - private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); - - @Test - void test() { - try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { - tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build()); - - ContainerName host1 = new ContainerName("host1"); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any()); - - tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED); - - tester.inOrder(tester.orchestrator).suspend(eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value()))); - tester.inOrder(tester.containerOperations).stopServices(containerMatcher(host1)); - assertTrue(tester.nodeAdmin.setFrozen(true)); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java deleted file mode 100644 index 1445546097a..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integration; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.integration.ContainerTester.containerMatcher; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; - -/** - * Tests that different wanted and current restart generation leads to execution of restart command - * - * @author musum - */ -public class RestartTest { - - @Test - void test() { - DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image:1.2.3"); - try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { - String hostname = "host1.test.yahoo.com"; - NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(dockerImage.tagAsVersion()) - .build(); - tester.addChildNodeRepositoryNode(nodeSpec); - - ContainerName host1 = new ContainerName("host1"); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any()); - tester.inOrder(tester.nodeRepository).updateNodeAttributes( - eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()))); - - // Increment wantedRestartGeneration to 2 in node-repo - tester.addChildNodeRepositoryNode(new NodeSpec.Builder(tester.nodeRepository.getNode(hostname)) - .wantedRestartGeneration(2).build()); - - tester.inOrder(tester.orchestrator).suspend(eq(hostname)); - tester.inOrder(tester.containerOperations).restartVespa(containerMatcher(host1)); - tester.inOrder(tester.nodeRepository).updateNodeAttributes( - eq(hostname), eq(new NodeAttributes().withRestartGeneration(2))); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java deleted file mode 100644 index 51b3bb5e6c4..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; -import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; - -/** - * @author dybis - */ -public class StorageMaintainerTest { - - private final TestTerminal terminal = new TestTerminal(); - private final CoredumpHandler coredumpHandler = mock(CoredumpHandler.class); - private final DiskCleanup diskCleanup = mock(DiskCleanup.class); - private final SyncClient syncClient = mock(SyncClient.class); - private final TestTimer timer = new TestTimer(Instant.ofEpochSecond(1234567890)); - private final FileSystem fileSystem = TestFileSystem.create(); - private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, timer, - fileSystem.getPath("/data/vespa/storage/container-archive")); - - @Test - void testDiskUsed() { - NodeAgentContext context = NodeAgentContextImpl.builder("host-1.domain.tld").fileSystem(fileSystem).build(); - - terminal.expectCommand("du -xsk /data/vespa/storage/host-1 2>&1", 0, "321\t/data/vespa/storage/host-1/"); - assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); - - // Value should still be cached, no new execution against the terminal - assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); - } - - @Test - void testNonExistingDiskUsed() { - DiskSize size = storageMaintainer.getDiskUsed(null, Path.of("/fake/path")); - assertEquals(DiskSize.ZERO, size); - } - - @Test - void archive_container_data_test() throws IOException { - // Create some files in containers - NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1"); - createNodeAgentContextAndContainerStorage(fileSystem, "container-2"); - - Path pathToArchiveDir = fileSystem.getPath("/data/vespa/storage/container-archive"); - Files.createDirectories(pathToArchiveDir); - - Path containerStorageRoot = context1.paths().of("/").pathOnHost().getParent(); - Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toSet()); - assertEquals(Set.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive); - - - // Archive container-1 - storageMaintainer.archiveNodeStorage(context1); - - timer.advance(Duration.ofSeconds(3)); - storageMaintainer.archiveNodeStorage(context1); - - // container-1 should be gone from container-storage - Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toSet()); - assertEquals(Set.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive); - - // container archive directory should contain exactly 1 directory - the one we just archived - List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list(); - assertEquals(1, containerArchiveContentsAfterArchive.size()); - Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path(); - assertEquals("container-1_20090213233130", archivedContainerStoragePath.getFileName().toString()); - Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath) - .stream() - .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString()) - .collect(Collectors.toSet()); - assertEquals(Set.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents); - } - - private static NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { - NodeAgentContext context = NodeAgentContextImpl.builder(containerName + ".domain.tld") - .fileSystem(fileSystem).build(); - - ContainerPath containerVespaHome = context.paths().underVespaHome(""); - Files.createDirectories(context.paths().of("/etc/something")); - Files.createFile(context.paths().of("/etc/something/conf")); - - Files.createDirectories(containerVespaHome.resolve("logs/vespa")); - Files.createFile(containerVespaHome.resolve("logs/vespa/vespa.log")); - Files.createFile(containerVespaHome.resolve("logs/vespa/zookeeper.log")); - - Files.createDirectories(containerVespaHome.resolve("var/db")); - Files.createFile(containerVespaHome.resolve("var/db/some-file")); - - Files.createDirectories(containerVespaHome.resolve("var/tmp")); - Files.createFile(containerVespaHome.resolve("var/tmp/some-file")); - - ContainerPath containerRoot = context.paths().of("/"); - Set<String> actualContents = FileFinder.files(containerRoot) - .stream() - .map(fileAttributes -> containerRoot.relativize(fileAttributes.path()).toString()) - .collect(Collectors.toSet()); - Set<String> expectedContents = Set.of( - "etc/something/conf", - "opt/vespa/logs/vespa/vespa.log", - "opt/vespa/logs/vespa/zookeeper.log", - "opt/vespa/var/tmp/some-file", - "opt/vespa/var/db/some-file"); - assertEquals(expectedContents, actualContents); - return context; - } - - @Test - void not_run_if_not_enough_used() { - NodeAgentContext context = NodeAgentContextImpl.builder( - NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) - .fileSystem(fileSystem).build(); - mockDiskUsage(500L); - - storageMaintainer.cleanDiskIfFull(context); - verifyNoInteractions(diskCleanup); - } - - @Test - void deletes_correct_amount() { - NodeAgentContext context = NodeAgentContextImpl.builder( - NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) - .fileSystem(fileSystem).build(); - - mockDiskUsage(950_000L); - - storageMaintainer.cleanDiskIfFull(context); - // Allocated size: 1 GB, usage: 950_000 kiB (972.8 MB). Wanted usage: 70% => 700 MB - verify(diskCleanup).cleanup(eq(context), any(), eq(272_800_000L)); - } - - @AfterEach - public void after() { - terminal.verifyAllCommandsExecuted(); - } - - private void mockDiskUsage(long kBytes) { - terminal.expectCommand("du -xsk /data/vespa/storage/h123a 2>&1", 0, kBytes + "\t/path"); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java deleted file mode 100644 index 063e8cb3f77..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.endsWith; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -public class AclMaintainerTest { - - private static final String EMPTY_FILTER_TABLE = "-P INPUT ACCEPT\n-P FORWARD ACCEPT\n-P OUTPUT ACCEPT\n"; - private static final String EMPTY_NAT_TABLE = "-P PREROUTING ACCEPT\n-P INPUT ACCEPT\n-P OUTPUT ACCEPT\n-P POSTROUTING ACCEPT\n"; - - private final ContainerOperations containerOperations = mock(ContainerOperations.class); - private final IPAddressesMock ipAddresses = new IPAddressesMock(); - private final AclMaintainer aclMaintainer = new AclMaintainer(containerOperations, ipAddresses); - - private final FileSystem fileSystem = TestFileSystem.create(); - private final Function<Acl, NodeAgentContext> contextGenerator = - acl -> NodeAgentContextImpl.builder("container1.host.com").fileSystem(fileSystem).acl(acl).build(); - private final List<String> writtenFileContents = new ArrayList<>(); - - @Test - void configures_full_container_acl_from_empty() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443) - .withTrustedNode("hostname1", "3001::abcd") - .withTrustedNode("hostname2", "3001::1234") - .withTrustedNode("hostname1", "192.168.0.5") - .withTrustedNode("hostname4", "172.16.5.234").build(); - NodeAgentContext context = contextGenerator.apply(acl); - - ipAddresses.addAddress(context.hostname().value(), "2001::1"); - ipAddresses.addAddress(context.hostname().value(), "10.0.0.1"); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); - whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); - whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); - - aclMaintainer.converge(context); - - verify(containerOperations, times(4)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); - verifyNoMoreInteractions(containerOperations); - - List<String> expected = List.of( - // IPv4 filter table restore - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -s 172.16.5.234/32 -j ACCEPT\n" + - "-A INPUT -s 192.168.0.5/32 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + - "COMMIT\n", - - // IPv6 filter table restore - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -s 3001::1234/128 -j ACCEPT\n" + - "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n" + - "COMMIT\n", - - // IPv4 nat table restore - "*nat\n" + - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 10.0.0.1/32 -j REDIRECT\n" + - "COMMIT\n", - - // IPv6 nat table restore - "*nat\n" + - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 2001::1/128 -j REDIRECT\n" + - "COMMIT\n"); - assertEquals(expected, writtenFileContents); - } - - @Test - void configures_minimal_container_acl_from_empty() { - // The ACL spec is empty and our this node's addresses do not resolve - Acl acl = new Acl.Builder().withTrustedPorts().build(); - NodeAgentContext context = contextGenerator.apply(acl); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); - whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); - whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); - - aclMaintainer.converge(context); - - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); - verifyNoMoreInteractions(containerOperations); - - List<String> expected = List.of( - // IPv4 filter table restore - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p icmp -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + - "COMMIT\n", - - // IPv6 filter table restore - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n" + - "COMMIT\n"); - assertEquals(expected, writtenFileContents); - } - - @Test - void only_configure_iptables_for_ipversion_that_differs() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build(); - NodeAgentContext context = contextGenerator.apply(acl); - - ipAddresses.addAddress(context.hostname().value(), "2001::1"); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n"); - whenListRules(context, "nat", IPVersion.IPv6, - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 2001::1/128 -j REDIRECT\n"); - - aclMaintainer.converge(context); - - verify(containerOperations, times(3)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, never()).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); //we don't have a ip4 address for the container so no redirect - verifyNoMoreInteractions(containerOperations); - - List<String> expected = List.of( - "*filter\n" + - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp-port-unreachable\n" + - "COMMIT\n"); - assertEquals(expected, writtenFileContents); - } - - @Test - void rollback_is_attempted_when_applying_acl_fail() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build(); - NodeAgentContext context = contextGenerator.apply(acl); - - ipAddresses.addAddress(context.hostname().value(), "2001::1"); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT\n" + - "-A INPUT -s 3001::abcd/128 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable\n"); - whenListRules(context, "nat", IPVersion.IPv6, - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 2001::1/128 -j REDIRECT\n"); - - when(containerOperations.executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any())) - .thenThrow(new RuntimeException("iptables restore failed")); - - aclMaintainer.converge(context); - - verify(containerOperations, times(3)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("iptables"), eq("-F"), eq("-t"), eq("filter")); - verifyNoMoreInteractions(containerOperations); - - aclMaintainer.converge(context); - } - - @Test - public void config_server_acl() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443) - .withTrustedNode("cfg1", "2001:db8::1") - .withTrustedNode("cfg2", "2001:db8::2") - .withTrustedNode("cfg3", "2001:db8::3") - .withTrustedNode("cfg1", "172.17.0.41") - .withTrustedNode("cfg2", "172.17.0.42") - .withTrustedNode("cfg3", "172.17.0.43") - .build(); - NodeAgentContext context = NodeAgentContextImpl.builder("cfg3.example.com") - .fileSystem(fileSystem) - .acl(acl) - .nodeSpecBuilder(builder -> builder.type(NodeType.config)) - .build(); - - ipAddresses.addAddress(context.hostname().value(), "2001:db8::3"); - ipAddresses.addAddress(context.hostname().value(), "172.17.0.43"); - - whenListRules(context, "filter", IPVersion.IPv4, EMPTY_FILTER_TABLE); - whenListRules(context, "filter", IPVersion.IPv6, EMPTY_FILTER_TABLE); - whenListRules(context, "nat", IPVersion.IPv4, EMPTY_NAT_TABLE); - whenListRules(context, "nat", IPVersion.IPv6, EMPTY_NAT_TABLE); - - aclMaintainer.converge(context); - - verify(containerOperations, times(4)).executeCommandInNetworkNamespace(eq(context), any(CommandLine.Options.class), any(), eq("-S"), eq("-t"), any()); - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("iptables-restore"), any()); - verify(containerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); - verifyNoMoreInteractions(containerOperations); - - List<String> expected = List.of( - // IPv4 filter table restore - """ - *filter - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 172.17.0.41/32 -j ACCEPT - -A INPUT -s 172.17.0.42/32 -j ACCEPT - -A INPUT -s 172.17.0.43/32 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp-port-unreachable - COMMIT - """, - // IPv6 filter table restore - """ - *filter - -P INPUT ACCEPT - -P FORWARD ACCEPT - -P OUTPUT ACCEPT - -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p ipv6-icmp -j ACCEPT - -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 2001:db8::1/128 -j ACCEPT - -A INPUT -s 2001:db8::2/128 -j ACCEPT - -A INPUT -s 2001:db8::3/128 -j ACCEPT - -A INPUT -j REJECT --reject-with icmp6-port-unreachable - COMMIT - """, - // IPv4 nat table restore - """ - *nat - -P PREROUTING ACCEPT - -P INPUT ACCEPT - -P OUTPUT ACCEPT - -P POSTROUTING ACCEPT - -A OUTPUT -d 172.17.0.43/32 -j REDIRECT - COMMIT - """, - // IPv6 nat table restore - """ - *nat - -P PREROUTING ACCEPT - -P INPUT ACCEPT - -P OUTPUT ACCEPT - -P POSTROUTING ACCEPT - -A OUTPUT -d 2001:db8::3/128 -j REDIRECT - COMMIT - """); - assertEquals(expected, writtenFileContents); - } - - @BeforeEach - public void setup() { - doAnswer(invoc -> { - String path = invoc.getArgument(2); - writtenFileContents.add(new UnixPath(path).readUtf8File()); - return new CommandResult(null, 0, ""); - }).when(containerOperations).executeCommandInNetworkNamespace(any(), endsWith("-restore"), any()); - } - - private void whenListRules(NodeAgentContext context, String table, IPVersion ipVersion, String output) { - when(containerOperations.executeCommandInNetworkNamespace( - eq(context), any(CommandLine.Options.class), eq(ipVersion.iptablesCmd()), eq("-S"), eq("-t"), eq(table))) - .thenReturn(new CommandResult(null, 0, output)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java deleted file mode 100644 index 52eac44fbc3..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class FilterTableLineEditorTest { - - @Test - void filter_set_wanted_rules() { - Acl acl = new Acl.Builder().withTrustedPorts(22).withTrustedNode("hostname", "3001::1").build(); - - assertFilterTableLineEditorResult( - acl, IPVersion.IPv6, - - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n", - - "-P INPUT ACCEPT\n" + - "-P FORWARD ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + - "-A INPUT -i lo -j ACCEPT\n" + - "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22 -j ACCEPT\n" + - "-A INPUT -s 3001::1/128 -j ACCEPT\n" + - "-A INPUT -j REJECT --reject-with icmp6-port-unreachable"); - } - - @Test - void produces_minimal_diff_simple() { - assertFilterTableDiff(List.of(2, 5, 3, 6, 1, 4), List.of(2, 5, 6, 1, 4), - "Patching file table:\n" + - "--A INPUT -s 2001::3/128 -j ACCEPT\n"); - } - - @Test - void produces_minimal_diff_complex() { - assertFilterTableDiff(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), List.of(5, 11, 6, 3, 10, 4, 8, 12), - "Patching file table:\n" + - "--A INPUT -s 2001::1/128 -j ACCEPT\n" + - "--A INPUT -s 2001::2/128 -j ACCEPT\n" + - "+-A INPUT -s 2001::11/128 -j ACCEPT\n" + - "+-A INPUT -s 2001::12/128 -j ACCEPT\n" + - "--A INPUT -s 2001::7/128 -j ACCEPT\n" + - "--A INPUT -s 2001::9/128 -j ACCEPT\n"); - } - - private static void assertFilterTableLineEditorResult( - Acl acl, IPVersion ipVersion, String currentFilterTable, String expectedRestoreFileContent) { - FilterTableLineEditor filterLineEditor = FilterTableLineEditor.from(acl, ipVersion); - Editor editor = new Editor( - "nat-table", - () -> List.of(currentFilterTable.split("\n")), - result -> assertEquals(expectedRestoreFileContent, String.join("\n", result)), - filterLineEditor); - editor.edit(m -> {}); - } - - private static void assertFilterTableDiff(List<Integer> currentIpSuffix, List<Integer> wantedIpSuffix, String diff) { - Acl.Builder currentAcl = new Acl.Builder(); - NodeType nodeType = NodeType.tenant; - currentIpSuffix.forEach(i -> currentAcl.withTrustedNode("host" + i, "2001::" + i)); - List<String> currentTable = new ArrayList<>(); - - Acl.Builder wantedAcl = new Acl.Builder(); - wantedIpSuffix.forEach(i -> wantedAcl.withTrustedNode("host" + i, "2001::" + i)); - - new Editor("table", List::of, currentTable::addAll, FilterTableLineEditor.from(currentAcl.build(), IPVersion.IPv6)) - .edit(log -> {}); - - new Editor("table", () -> currentTable, result -> {}, FilterTableLineEditor.from(wantedAcl.build(), IPVersion.IPv6)) - .edit(log -> assertEquals(diff, log)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java deleted file mode 100644 index d8d526050d7..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class NatTableLineEditorTest { - - @Test - void nat_set_redirect_rule_without_touching_docker_rules() { - assertNatTableLineEditorResult( - "-A OUTPUT -d 3001::1/128 -j REDIRECT", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-N DOCKER_OUTPUT\n" + - "-N DOCKER_POSTROUTING\n" + - "-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n" + - "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n" + - "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:43500\n" + - "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:57392\n" + - "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 43500 -j SNAT --to-source :53\n" + - "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 57392 -j SNAT --to-source :53\n", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-N DOCKER_OUTPUT\n" + - "-N DOCKER_POSTROUTING\n" + - "-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n" + - "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n" + - "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:43500\n" + - "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:57392\n" + - "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 43500 -j SNAT --to-source :53\n" + - "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 57392 -j SNAT --to-source :53\n" + - "-A OUTPUT -d 3001::1/128 -j REDIRECT"); - } - - @Test - void nat_cleanup_wrong_redirect_rules() { - assertNatTableLineEditorResult( - "-A OUTPUT -d 3001::1/128 -j REDIRECT", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 3001::2/128 -j REDIRECT\n", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 3001::1/128 -j REDIRECT"); - } - - @Test - void nat_delete_duplicate_rules() { - assertNatTableLineEditorResult( - "-A OUTPUT -d 3001::1/128 -j REDIRECT", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 3001::2/128 -j REDIRECT\n" + - "-A OUTPUT -d 3001::1/128 -j REDIRECT\n" + - "-A OUTPUT -d 3001::4/128 -j REDIRECT\n", - - "-P PREROUTING ACCEPT\n" + - "-P INPUT ACCEPT\n" + - "-P OUTPUT ACCEPT\n" + - "-P POSTROUTING ACCEPT\n" + - "-A OUTPUT -d 3001::1/128 -j REDIRECT"); - } - - private static void assertNatTableLineEditorResult(String redirectRule, String currentNatTable, String expectedNatTable) { - NatTableLineEditor natLineEditor = NatTableLineEditor.from(redirectRule); - Editor editor = new Editor( - "nat-table", - () -> List.of(currentNatTable.split("\n")), - result -> assertEquals(expectedNatTable, String.join("\n", result)), - natLineEditor); - editor.edit(m -> {}); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java deleted file mode 100644 index 9487affd376..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; - -import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class CoreCollectorTest { - private static final Instant CORE_CREATED = Instant.ofEpochMilli(2233445566L); - - private final ContainerOperations docker = mock(ContainerOperations.class); - private final CoreCollector coreCollector = new CoreCollector(docker); - private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") - .fileSystem(TestFileSystem.create()).build(); - - private final ContainerPath TEST_CORE_PATH = (ContainerPath) new UnixPath(context.paths().of("/tmp/core.1234")) - .createParents() - .createNewFile() - .setLastModifiedTime(CORE_CREATED) - .toPath(); - private final String TEST_BIN_PATH = "/usr/bin/program"; - private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]", - "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", - "#0 0x00000000004004d8 in main (argv=...) at main.c:4", "4\t printf(argv[3]);", - "#0 0x00000000004004d8 in main (argv=...) at main.c:4"); - - @Test - void extractsBinaryPathTest() { - final String[] cmd = {"file", TEST_CORE_PATH.pathInContainer()}; - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program --foo --bar baz'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, " + - "from 'program', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, " + - "execfn: '/usr/bin/program', platform: 'x86_64"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); - - String fallbackResponse = "/response/from/fallback"; - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}, - """ - GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1 - Type “apropos word” to search for commands related to “word”… - Reading symbols from abc…(no debugging symbols found)…done. - [New LWP 23678] - Core was generated by `/response/from/fallback'. \s - Program terminated with signal SIGSEGV, Segmentation fault. \s - #0 0x0000000000400541 in main () - #0 0x0000000000400541 in main () - (gdb) bt - #0 0x0000000000400541 in main () - (gdb) - """); - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style"); - assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH)); - - mockExec(cmd, "", "Error code 1234"); - assertEquals(fallbackResponse, coreCollector.readBinPath(context, TEST_CORE_PATH)); - } - - @Test - void extractsBinaryPathUsingGdbTest() { - String[] cmd = new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}; - - mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'."); - assertEquals("/usr/bin/program-from-gdb", coreCollector.readBinPathFallback(context, TEST_CORE_PATH)); - - mockExec(cmd, "", "Error 123"); - try { - coreCollector.readBinPathFallback(context, TEST_CORE_PATH); - fail("Expected not to be able to get bin path"); - } catch (RuntimeException e) { - assertEquals("Failed to extract binary path from GDB, result: exit status 1, output 'Error 123', command: " + - "[/opt/rh/gcc-toolset-12/root/bin/gdb, -n, -batch, -core, /tmp/core.1234]", e.getMessage()); - } - } - - @Test - void extractsBacktraceUsingGdb() { - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false)); - - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - "", "Failure"); - try { - coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, false); - fail("Expected not to be able to read backtrace"); - } catch (RuntimeException e) { - assertEquals("Failed to read backtrace exit status 1, output 'Failure', Command: " + - "[" + GDB_PATH_RHEL8 + ", -n, -ex, set print frame-arguments none, -ex, bt, -batch, " + - "/usr/bin/program, /tmp/core.1234]", e.getMessage()); - } - } - - @Test - void extractsBacktraceFromAllThreadsUsingGdb() { - mockExec(new String[]{GDB_PATH_RHEL8, "-n", - "-ex", "set print frame-arguments none", - "-ex", "thread apply all bt", "-batch", - "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(context, TEST_CORE_PATH, TEST_BIN_PATH, true)); - } - - @Test - void collectsDataTest() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "thread apply all bt", "-batch", - "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - - var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH) - .setCreated(CORE_CREATED) - .setType(CoreDumpMetadata.Type.CORE_DUMP) - .setBacktrace(GDB_BACKTRACE) - .setBacktraceAllThreads(GDB_BACKTRACE); - assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); - } - - @Test - void collectsDataRelativePath() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'sbin/distributord-bin'"); - String absolutePath = "/opt/vespa/sbin/distributord-bin"; - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "bt", "-batch", absolutePath, "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-ex", "set print frame-arguments none", - "-ex", "thread apply all bt", "-batch", absolutePath, "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - - var expected = new CoreDumpMetadata() - .setBinPath(absolutePath) - .setCreated(CORE_CREATED) - .setType(CoreDumpMetadata.Type.CORE_DUMP) - .setBacktrace(GDB_BACKTRACE) - .setBacktraceAllThreads(GDB_BACKTRACE); - assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); - } - - @Test - void collectsPartialIfBacktraceFailsTest() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH_RHEL8 + " -n -ex set print frame-arguments none -ex bt -batch /usr/bin/program /tmp/core.1234"}, - "", "Failure"); - - var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH).setCreated(CORE_CREATED).setType(CoreDumpMetadata.Type.CORE_DUMP); - assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); - } - - @Test - void reportsJstackInsteadOfGdbForJdkCores() { - mockExec(new String[]{"file", TEST_CORE_PATH.pathInContainer()}, - "dump.core.5954: ELF 64-bit LSB core file x86-64, version 1 (SYSV), too many program header sections (33172)"); - - String jdkPath = "/path/to/jdk/java"; - mockExec(new String[]{GDB_PATH_RHEL8, "-n", "-batch", "-core", "/tmp/core.1234"}, - "Core was generated by `" + jdkPath + " -Dconfig.id=default/container.11 -XX:+Pre'."); - - String jstack = "jstack11"; - mockExec(new String[]{"jhsdb", "jstack", "--exe", jdkPath, "--core", "/tmp/core.1234"}, - jstack); - - var expected = new CoreDumpMetadata().setBinPath(jdkPath) - .setCreated(CORE_CREATED) - .setType(CoreDumpMetadata.Type.CORE_DUMP) - .setBacktraceAllThreads(List.of(jstack)); - assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); - } - - @Test - void metadata_for_java_heap_dump() { - var expected = new CoreDumpMetadata().setBinPath("java") - .setType(CoreDumpMetadata.Type.JVM_HEAP) - .setCreated(CORE_CREATED) - .setBacktrace(List.of("Heap dump, no backtrace available")); - - assertEquals(expected, coreCollector.collect(context, (ContainerPath) new UnixPath(context.paths().of("/dump_java_pid123.hprof")) - .createNewFile() - .setLastModifiedTime(CORE_CREATED) - .toPath())); - } - - private void mockExec(String[] cmd, String output) { - mockExec(cmd, output, ""); - } - - private void mockExec(String[] cmd, String output, String error) { - mockExec(context, cmd, output, error); - } - - private void mockExec(NodeAgentContext context, String[] cmd, String output, String error) { - when(docker.executeCommandInContainer(context, context.users().root(), cmd)) - .thenReturn(new CommandResult(null, error.isEmpty() ? 0 : 1, error.isEmpty() ? output : error)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java deleted file mode 100644 index e65a226b789..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.security.KeyId; -import com.yahoo.security.SealedSharedKey; -import com.yahoo.security.SecretSharedKey; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; -import com.yahoo.vespa.hosted.node.admin.configserver.cores.Cores; -import com.yahoo.vespa.hosted.node.admin.container.metrics.DimensionMetrics; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.crypto.spec.SecretKeySpec; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class CoredumpHandlerTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") - .fileSystem(fileSystem).build(); - private final ContainerPath containerCrashPath = context.paths().of("/var/crash"); - private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); - - private final CoreCollector coreCollector = mock(CoreCollector.class); - private final Cores cores = mock(Cores.class); - private final Metrics metrics = new Metrics(); - private final TestTimer timer = new TestTimer(); - @SuppressWarnings("unchecked") - private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); - private final SecretSharedKeySupplier secretSharedKeySupplier = mock(SecretSharedKeySupplier.class); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private final CoredumpHandler coredumpHandler = - new CoredumpHandler(coreCollector, cores, containerCrashPath.pathInContainer(), - doneCoredumpsPath, metrics, timer, coredumpIdSupplier, secretSharedKeySupplier, - flagSource); - - @Test - void coredump_enqueue_test() throws IOException { - ContainerPath crashPath = context.paths().of("/some/crash/path"); - ContainerPath processingDir = context.paths().of("/some/other/processing"); - - Files.createDirectories(crashPath); - createFileAged(crashPath.resolve("bash.core.431"), Duration.ZERO); - - assertFolderContents(crashPath, "bash.core.431"); - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); - assertEquals(Optional.empty(), enqueuedPath); - - // bash.core.431 finished writing... and 2 more have since been written - timer.advance(Duration.ofMinutes(3)); - createFileAged(crashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); - createFileAged(crashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); - - when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); - enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - assertFolderContents(crashPath, "bash.core.431", "vespa-slobrok.core.673"); - assertFolderContents(processingDir, "id-123"); - assertFolderContents(processingDir.resolve("id-123"), "dump_vespa-proton.core.119"); - verify(coredumpIdSupplier, times(1)).get(); - - // Enqueue another - enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-321")), enqueuedPath); - assertFolderContents(crashPath, "bash.core.431"); - assertFolderContents(processingDir, "id-123", "id-321"); - assertFolderContents(processingDir.resolve("id-321"), "dump_vespa-slobrok.core.673"); - verify(coredumpIdSupplier, times(2)).get(); - } - - @Test - void enqueue_with_hs_err_files() throws IOException { - ContainerPath crashPath = context.paths().of("/some/crash/path"); - ContainerPath processingDir = context.paths().of("/some/other/processing"); - Files.createDirectories(crashPath); - - createFileAged(crashPath.resolve("java.core.69"), Duration.ofSeconds(515)); - createFileAged(crashPath.resolve("hs_err_pid69.log"), Duration.ofSeconds(520)); - - createFileAged(crashPath.resolve("java.core.2420"), Duration.ofSeconds(540)); - createFileAged(crashPath.resolve("hs_err_pid2420.log"), Duration.ofSeconds(549)); - createFileAged(crashPath.resolve("hs_err_pid2421.log"), Duration.ofSeconds(550)); - - when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, crashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - assertFolderContents(crashPath, "hs_err_pid69.log", "java.core.69"); - assertFolderContents(processingDir, "id-123"); - assertFolderContents(processingDir.resolve("id-123"), "hs_err_pid2420.log", "hs_err_pid2421.log", "dump_java.core.2420"); - } - - @Test - void coredump_to_process_test() throws IOException { - ContainerPath processingDir = context.paths().of("/some/other/processing"); - - // Initially there are no core dumps - Optional<ContainerPath> enqueuedPath = coredumpHandler.enqueueCoredump(context, containerCrashPath, processingDir); - assertEquals(Optional.empty(), enqueuedPath); - - // 3 core dumps occur - Files.createDirectories(containerCrashPath); - createFileAged(containerCrashPath.resolve("bash.core.431"), Duration.ZERO); - createFileAged(containerCrashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); - createFileAged(containerCrashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); - - when(coredumpIdSupplier.get()).thenReturn("id-123"); - enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - - // Running this again wont enqueue new core dumps as we are still processing the one enqueued previously - enqueuedPath = coredumpHandler.getCoredumpToProcess(context, containerCrashPath, processingDir); - assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); - verify(coredumpIdSupplier, times(1)).get(); - } - - @Test - void gather_metadata_test() throws IOException { - var metadata = new CoreDumpMetadata().setKernelVersion("3.10.0-862.9.1.el7.x86_64") - .setBacktrace(List.of("call 1", "function 2", "something something")) - .setVespaVersion("6.48.4") - .setBinPath("/bin/bash") - .setCoreDumpPath(context.paths().of("/home/docker/dumps/container-123/id-123/dump_core.456")) - .setDockerImage(DockerImage.fromString("example.com/vespa/ci:6.48.4")); - - new UnixPath(fileSystem.getPath("/proc/cpuinfo")).createParents().writeUtf8File("microcode\t: 0xf0"); - - ContainerPath coredumpDirectory = context.paths().of("/var/crash/id-123"); - Files.createDirectories(coredumpDirectory.pathOnHost()); - Files.createFile(coredumpDirectory.resolve("dump_core.456")); - when(coreCollector.collect(eq(context), eq(coredumpDirectory.resolve("dump_core.456")))) - .thenReturn(metadata); - - assertEquals(metadata, coredumpHandler.gatherMetadata(context, coredumpDirectory)); - verify(coreCollector, times(1)).collect(any(), any()); - - // On second invocation the test already exist, so times(1) is not incremented - assertEquals(metadata, coredumpHandler.gatherMetadata(context, coredumpDirectory)); - doThrow(new IllegalStateException("Should not be invoked")) - .when(coreCollector).collect(any(), any()); - verify(coreCollector, times(1)).collect(any(), any()); - } - - @Test - void cant_get_metadata_if_no_core_file() { - assertThrows(IllegalStateException.class, () -> { - coredumpHandler.gatherMetadata(context, context.paths().of("/fake/path")); - }); - } - - @Test - void fails_to_get_core_file_if_only_compressed_or_encrypted() { - assertThrows(IllegalStateException.class, () -> { - ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); - Files.createDirectories(coredumpDirectory); - Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.zst")); - Files.createFile(coredumpDirectory.resolve("dump_bash.core.543.zst.enc")); - coredumpHandler.findCoredumpFileInProcessingDirectory(coredumpDirectory); - }); - } - - void do_process_single_coredump_test(String expectedCoreFileName) throws IOException { - ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); - Files.createDirectories(coredumpDirectory); - Files.write(coredumpDirectory.resolve("metadata2.json"), "{\"test-metadata\":{}}".getBytes()); - Files.createFile(coredumpDirectory.resolve("dump_bash.core.431")); - assertFolderContents(coredumpDirectory, "metadata2.json", "dump_bash.core.431"); - CoreDumpMetadata expectedMetadata = new CoreDumpMetadata(); - expectedMetadata.setDecryptionToken("131Q0MMF3hBuMVnXg1WnSFexZGrcwa9ZhfHlegLNwPIN6hQJnBxq5srLf3aZbYdlRVE"); - - coredumpHandler.processAndReportSingleCoreDump(context, coredumpDirectory, Optional.empty()); - verify(coreCollector, never()).collect(any(), any()); - verify(cores, times(1)).report(eq(context.hostname()), eq("id-123"), eq(expectedMetadata)); - assertFalse(Files.exists(coredumpDirectory)); - assertFolderContents(doneCoredumpsPath.resolve("container-123"), "id-123"); - assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata2.json", expectedCoreFileName); - } - - @Test - void processing_single_coredump_test_without_encryption_throws() throws IOException { - assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); - } - - @Test - void process_single_coredump_test_with_encryption() throws IOException { - flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "bar-key"); - when(secretSharedKeySupplier.create(KeyId.ofString("bar-key"))).thenReturn(Optional.of(makeFixedSecretSharedKey())); - do_process_single_coredump_test("dump_bash.core.431.zst.enc"); - } - - @Test - void processing_throws_when_no_public_key_set_in_feature_flag() throws IOException { - flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), ""); // empty -> not set - verify(secretSharedKeySupplier, never()).create(any()); - assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); - } - - @Test - void processing_throws_when_no_key_returned_for_key_id_specified_by_feature_flag() throws IOException { - flagSource.withStringFlag(Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.id(), "baz-key"); - when(secretSharedKeySupplier.create(KeyId.ofString("baz-key"))).thenReturn(Optional.empty()); - assertThrows(ConvergenceException.class, () -> do_process_single_coredump_test("dump_bash.core.431.zst")); - } - - @Test - void report_enqueued_and_processed_metrics() throws IOException { - Path processingPath = containerCrashPath.resolve("processing"); - Files.createFile(containerCrashPath.resolve("dump-1")); - Files.createFile(containerCrashPath.resolve("dump-2")); - Files.createFile(containerCrashPath.resolve("hs_err_pid2.log")); - Files.createDirectory(processingPath); - Files.createFile(processingPath.resolve("metadata2.json")); - Files.createFile(processingPath.resolve("dump-3")); - - new UnixPath(doneCoredumpsPath.resolve("container-123").resolve("dump-3-folder").resolve("dump-3")) - .createParents() - .createNewFile(); - - coredumpHandler.updateMetrics(context, containerCrashPath); - List<DimensionMetrics> updatedMetrics = metrics.getMetricsByType(Metrics.DimensionType.PRETAGGED); - assertEquals(1, updatedMetrics.size()); - Map<String, Number> values = updatedMetrics.get(0).getMetrics(); - assertEquals(3, values.get("coredumps.enqueued").intValue()); - assertEquals(1, values.get("coredumps.processed").intValue()); - } - - @BeforeEach - public void setup() throws IOException { - Files.createDirectories(containerCrashPath.pathOnHost()); - } - - private static void assertFolderContents(Path pathToFolder, String... filenames) { - Set<String> expectedContentsOfFolder = Set.of(filenames); - Set<String> actualContentsOfFolder; - try (Stream<UnixPath> paths = new UnixPath(pathToFolder).listContentsOfDirectory()) { - actualContentsOfFolder = paths.map(unixPath -> unixPath.toPath().getFileName().toString()) - .collect(Collectors.toSet()); - } - assertEquals(expectedContentsOfFolder, actualContentsOfFolder); - } - - private Path createFileAged(Path path, Duration age) { - return uncheck(() -> Files.setLastModifiedTime( - Files.createFile(path), - FileTime.from(timer.currentTime().minus(age)))); - } - - private static byte[] bytesOf(String str) { - return str.getBytes(StandardCharsets.UTF_8); - } - - private static SecretSharedKey makeFixedSecretSharedKey() { - byte[] keyBytes = bytesOf("very secret yes!"); // 128 bits - var secretKey = new SecretKeySpec(keyBytes, "AES"); - var keyId = KeyId.ofString("the shiniest key"); - // We don't parse any of these fields in the test, so just use dummy contents. - byte[] enc = bytesOf("hello world"); - byte[] ciphertext = bytesOf("imaginary ciphertext"); - return new SecretSharedKey(secretKey, new SealedSharedKey(SealedSharedKey.CURRENT_TOKEN_VERSION, keyId, enc, ciphertext)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java deleted file mode 100644 index 0e20d3965a0..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Instant; -import java.util.Map; -import java.util.TreeMap; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class CoredumpCleanupRuleTest { - - private final FileSystem fileSystem = TestFileSystem.create(); - - @Test - void for_container_test() throws IOException { - Path path = fileSystem.getPath("/test/path"); - DiskCleanupRule rule = CoredumpCleanupRule.forContainer(path); - - assertPriorities(rule, Map.of()); - - createFile(path.resolve("core1"), Instant.ofEpochSecond(232)); - assertPriorities(rule, Map.of("/test/path/core1", Priority.MEDIUM)); - - createFile(path.resolve("core2"), Instant.ofEpochSecond(123)); - assertPriorities(rule, Map.of( - "/test/path/core2", Priority.MEDIUM, - "/test/path/core1", Priority.HIGHEST)); - - createFile(path.resolve("vespa-proton-bin.core.325"), Instant.ofEpochSecond(456)); - createFile(path.resolve("vespa-distributor.core.764"), Instant.ofEpochSecond(256)); - var expected = Map.of( - "/test/path/core2", Priority.HIGHEST, - "/test/path/core1", Priority.HIGHEST, - "/test/path/vespa-proton-bin.core.325", Priority.HIGHEST, - "/test/path/vespa-distributor.core.764", Priority.MEDIUM); - assertPriorities(rule, expected); - - // processing core has no effect on this - Files.createDirectories(path.resolve("processing/abcd-1234")); - createFile(path.resolve("processing/abcd-1234/core5"), Instant.ofEpochSecond(67)); - assertPriorities(rule, expected); - } - - @Test - void for_host_test() throws IOException { - Path path = fileSystem.getPath("/test/path"); - DiskCleanupRule rule = CoredumpCleanupRule.forHost(path); - - assertPriorities(rule, Map.of()); - - createFile(path.resolve("h123a/abcd-1234/dump_core1"), Instant.parse("2020-04-21T19:21:00Z")); - createFile(path.resolve("h123a/abcd-1234/metadata.json"), Instant.parse("2020-04-21T19:26:00Z")); - assertPriorities(rule, Map.of("/test/path/h123a/abcd-1234/dump_core1", Priority.MEDIUM)); - - createFile(path.resolve("h123a/abcd-efgh/dump_core1"), Instant.parse("2020-04-21T07:13:00Z")); - createFile(path.resolve("h123a/56ad-af42/dump_vespa-distributor.321"), Instant.parse("2020-04-21T23:37:00Z")); - createFile(path.resolve("h123a/4324-a23d/dump_core2"), Instant.parse("2020-04-22T04:56:00Z")); - createFile(path.resolve("h123a/8534-7da3/dump_vespa-proton-bin.123"), Instant.parse("2020-04-19T15:35:00Z")); - - // Also create a core for a second container: h123b - createFile(path.resolve("h123b/db1a-ab34/dump_core1"), Instant.parse("2020-04-21T07:01:00Z")); - createFile(path.resolve("h123b/7392-59ad/dump_vespa-proton-bin.342"), Instant.parse("2020-04-22T12:05:00Z")); - - assertPriorities(rule, Map.of( - "/test/path/h123a/abcd-1234/dump_core1", Priority.HIGH, - "/test/path/h123a/abcd-efgh/dump_core1", Priority.HIGH, - - // Although it is the oldest core of the day for h123a, it is the first one that starts with vespa- - "/test/path/h123a/56ad-af42/dump_vespa-distributor.321", Priority.MEDIUM, - "/test/path/h123a/4324-a23d/dump_core2", Priority.MEDIUM, - "/test/path/h123a/8534-7da3/dump_vespa-proton-bin.123", Priority.MEDIUM, - "/test/path/h123b/db1a-ab34/dump_core1", Priority.MEDIUM, - "/test/path/h123b/7392-59ad/dump_vespa-proton-bin.342", Priority.MEDIUM - )); - } - - private static void createFile(Path path, Instant instant) throws IOException { - Files.createDirectories(path.getParent()); - Files.createFile(path); - Files.setLastModifiedTime(path, FileTime.from(instant)); - } - - private static void assertPriorities(DiskCleanupRule rule, Map<String, Priority> expected) { - Map<String, Priority> actual = rule.prioritize().stream() - .collect(Collectors.toMap(pfa -> pfa.fileAttributes().path().toString(), PrioritizedFileAttributes::priority)); - - assertEquals(new TreeMap<>(expected), new TreeMap<>(actual)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java deleted file mode 100644 index 390501a4530..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.PosixFileAttributeView; -import java.nio.file.attribute.PosixFileAttributes; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - */ -public class DiskCleanupTest { - - private final TestTaskContext context = new TestTaskContext(); - private final DiskCleanupTester tester = new DiskCleanupTester(); - private final DiskCleanup diskCleanup = new DiskCleanup(); - - @Test - void nothing_deleted() throws IOException { - assertFalse(diskCleanup.cleanup(context, List.of(), 0)); - assertFalse(diskCleanup.cleanup(context, List.of(), 10)); - - DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock(); - DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock(); - assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 0)); - assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - - tester.createFile("/path/that-should-not-be-deleted", 5); - assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - tester.assertAllFilesExistExcept(); - - // Create a file and let rule return it, but before cleanup is run, the file is deleted - rule1.addFile(tester.createFile("/path/file-does-not-exist", 1), Priority.HIGHEST); - Files.delete(tester.path("/path/file-does-not-exist")); - assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - } - - @Test - void delete_test() throws IOException { - tester.createFile("/opt/vespa/var/db/do-not-delete-1.db", 1); - tester.createFile("/opt/vespa/var/db/do-not-delete-2.db", 1); - tester.createFile("/opt/vespa/var/zookeeper/do-not-delete-3", 1); - tester.createFile("/opt/vespa/var/index/something-important", 1); - - DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock() - .addFile(tester.createFile("/opt/vespa/logs/vespa-1.log", 10), Priority.MEDIUM) - .addFile(tester.createFile("/opt/vespa/logs/vespa-2.log", 8), Priority.HIGH) - .addFile(tester.createFile("/opt/vespa/logs/vespa-3.log", 13), Priority.HIGHEST) - .addFile(tester.createFile("/opt/vespa/logs/vespa-4.log", 10), Priority.HIGHEST); - DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock() - .addFile(tester.createFile("/opt/vespa/var/crash/core1", 105), Priority.LOW) - .addFile(tester.createFile("/opt/vespa/var/crash/vespa-proton-bin.core-232", 190), Priority.HIGH) - .addFile(tester.createFile("/opt/vespa/var/crash/core3", 54), Priority.MEDIUM) - .addFile(tester.createFile("/opt/vespa/var/crash/core4", 300), Priority.HIGH); - - // 2 files with HIGHEST priority, tie broken by the largest size which is won by "vespa-3.log", since - // it is >= 10 bytes, no more files are deleted - assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log"); - - // Called with the same arguments, but vespa-3.log is still missing... - assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); - tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log"); - - assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 500)); - tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log", // from before - // 300 + 190 + 8 + 54 - "/opt/vespa/var/crash/core4", "/opt/vespa/var/crash/vespa-proton-bin.core-232", "/opt/vespa/logs/vespa-2.log", "/opt/vespa/var/crash/core3"); - } - - private static class DiskCleanupRuleMock implements DiskCleanupRule { - private final ArrayList<PrioritizedFileAttributes> pfa = new ArrayList<>(); - - private DiskCleanupRuleMock addFile(Path path, Priority priority) throws IOException { - PosixFileAttributes attributes = Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes(); - pfa.add(new PrioritizedFileAttributes(new FileAttributes(path, attributes), priority)); - return this; - } - - @Override - public Collection<PrioritizedFileAttributes> prioritize() { - return Collections.unmodifiableList(pfa); - } - } - - private static class DiskCleanupTester { - private final FileSystem fileSystem = TestFileSystem.create(); - private final Set<String> files = new HashSet<>(); - - private Path path(String path) { - return fileSystem.getPath(path); - } - - private Path createFile(String pathStr, int size) throws IOException { - Path path = path(pathStr); - Files.createDirectories(path.getParent()); - Files.write(path, new byte[size]); - files.add(path.toString()); - return path; - } - - private void assertAllFilesExistExcept(String... deletedPaths) { - Set<String> actual = FileFinder.files(path("/")).stream().map(fa -> fa.path().toString()).collect(Collectors.toSet()); - Set<String> expected = new HashSet<>(files); - expected.removeAll(Set.of(deletedPaths)); - assertEquals(expected, actual); - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java deleted file mode 100644 index c85ddf41906..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.disk; - -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; -import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; -import static org.mockito.Mockito.mock; - -/** - * @author freva - */ -public class LinearCleanupRuleTest { - - @Test - void basic() { - assertRule(Map.of(), Priority.LOWEST, Priority.HIGHEST); - - assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.LOW, 1.0, Priority.LOW), Priority.LOW, Priority.LOW); - assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.MEDIUM, 1.0, Priority.MEDIUM), Priority.LOW, Priority.MEDIUM); - - assertRule(Map.of( - -5.0, Priority.LOW, - 0.0, Priority.LOW, - 0.2, Priority.LOW, - 0.35, Priority.MEDIUM, - 0.65, Priority.MEDIUM, - 0.8, Priority.HIGH, - 1.0, Priority.HIGH, - 5.0, Priority.HIGH), - Priority.LOW, Priority.HIGH); - } - - @Test - void fail_if_high_priority_lower_than_low() { - assertThrows(IllegalArgumentException.class, () -> { - assertRule(Map.of(), Priority.HIGHEST, Priority.LOWEST); - }); - } - - private static void assertRule(Map<Double, Priority> expectedPriorities, Priority low, Priority high) { - Map<FileAttributes, Double> fileAttributesByScore = expectedPriorities.keySet().stream() - .collect(Collectors.toMap(score -> mock(FileAttributes.class), score -> score)); - LinearCleanupRule rule = new LinearCleanupRule( - () -> List.copyOf(fileAttributesByScore.keySet()), fileAttributesByScore::get, low, high); - - Map<Double, Priority> actualPriorities = rule.prioritize().stream() - .collect(Collectors.toMap(pfa -> fileAttributesByScore.get(pfa.fileAttributes()), PrioritizedFileAttributes::priority)); - assertEquals(expectedPriorities, actualPriorities); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java deleted file mode 100644 index 607efa9771a..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ArtifactProducersTest.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.yahoo.yolean.concurrent.Sleeper; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author bjorncs - */ -class ArtifactProducersTest { - - @Test - void generates_exception_on_unknown_artifact() { - ArtifactProducers instance = ArtifactProducers.createDefault(Sleeper.NOOP); - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, () -> instance.resolve(List.of("unknown-artifact"))); - String expectedMsg = - "Invalid artifact type 'unknown-artifact'. Valid types are " + - "['config-dump', 'jvm-heap-dump', 'jvm-jfr', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'perf-report', 'pmap', " + - "'vespa-log', 'zookeeper-snapshot'] " + - "and valid aliases are " + - "['jvm-dump': ['jvm-heap-dump', 'jvm-jmap', 'jvm-jstack', 'jvm-jstat', 'vespa-log']]"; - assertEquals(expectedMsg, exception.getMessage()); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java deleted file mode 100644 index db19d6b0074..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.integration.NodeRepoMock; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncClient; -import com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; -import com.yahoo.vespa.test.file.TestFileSystem; -import com.yahoo.yolean.concurrent.Sleeper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.List; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.ServiceDumpReport.DumpOptions; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author bjorncs - */ -class VespaServiceDumperImplTest { - - private static final String HOSTNAME = "host-1.domain.tld"; - - private final FileSystem fileSystem = TestFileSystem.create(); - private final Path tmpDirectory = fileSystem.getPath("/data/vespa/storage/host-1/opt/vespa/var/tmp"); - - @BeforeEach - void create_tmp_directory() throws IOException { - // Create temporary directory in container - Files.createDirectories(tmpDirectory); - } - - @Test - void creates_valid_dump_id_from_dump_request() { - long nowMillis = Instant.now().toEpochMilli(); - ServiceDumpReport request = new ServiceDumpReport( - nowMillis, null, null, null, null, "default/container.3", null, null, List.of("perf-report"), null); - String dumpId = VespaServiceDumperImpl.createDumpId(request); - assertEquals("default-container-3-" + nowMillis, dumpId); - } - - @Test - void invokes_perf_commands_when_generating_perf_report() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - .thenReturn(new CommandResult(null, 0, "12345")) - .thenReturn(new CommandResult(null, 0, "")) - .thenReturn(new CommandResult(null, 0, "")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report"), new ServiceDumpReport.DumpOptions(true, 45.0, null)); - - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "perf", "record", "-g", "--output=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-record.bin", - "--pid=12345", "sleep", "45"); - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "bash", "-c", "perf report --input=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-record.bin" + - " > /opt/vespa/var/tmp/vespa-service-dump-1600000000000/perf-report.txt"); - - String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + - "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + - "\"configId\":\"default/container.1\",\"artifacts\":[\"perf-report\"]," + - "\"dumpOptions\":{\"callGraphRecording\":true,\"duration\":45.0}}"; - assertReportEquals(nodeRepository, expectedJson); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-record.bin.zst"), - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-report.txt")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void invokes_jcmd_commands_when_creating_jfr_recording() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - .thenReturn(new CommandResult(null, 0, "12345")) - .thenReturn(new CommandResult(null, 0, "ok")) - .thenReturn(new CommandResult(null, 0, "name=host-admin success")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("jvm-jfr")); - - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "/opt/vespa/libexec/vespa/find-pid", "default/container.1"); - verify(operations).executeCommandInContainer( - context, context.users().vespa(), "jcmd", "12345", "JFR.start", "name=host-admin", "path-to-gc-roots=true", "settings=profile", - "filename=/opt/vespa/var/tmp/vespa-service-dump-1600000000000/recording.jfr", "duration=30s"); - verify(operations).executeCommandInContainer(context, context.users().vespa(), "jcmd", "12345", "JFR.check", "name=host-admin"); - - String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000," + - "\"completedAt\":1600001000000," + - "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + - "\"configId\":\"default/container.1\",\"artifacts\":[\"jvm-jfr\"],\"dumpOptions\":{}}"; - assertReportEquals(nodeRepository, expectedJson); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/recording.jfr.zst")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void invokes_zookeeper_backup_command_when_generating_snapshot() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - .thenReturn(new CommandResult(null, 0, "12345")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("zookeeper-snapshot")); - - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - verify(operations).executeCommandInContainer( - context, - context.users().vespa(), - "bash", - "-c", - "/opt/vespa/bin/vespa-backup-zk-data.sh -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/zookeeper-snapshot.tgz -k -f"); - - String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + - "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + - "\"configId\":\"default/container.1\",\"artifacts\":[\"zookeeper-snapshot\"],\"dumpOptions\":{}}"; - assertReportEquals(nodeRepository, expectedJson); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/zookeeper-snapshot.tgz")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void invokes_config_proxy_command_whn_invoking_config_dump() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer(any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - .thenReturn(new CommandResult(null, 0, "12345")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("config-dump")); - - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - verify(operations).executeCommandInContainer( - context, - context.users().vespa(), - "bash", - "-c", - "mkdir -p /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + - " /opt/vespa/bin/vespa-configproxy-cmd -m dumpcache /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + - " tar cvf /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config.tar /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config;" + - " zstd /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config.tar -o /opt/vespa/var/tmp/vespa-service-dump-1600000000000/config-dump.tar.zst"); - - String expectedJson = "{\"createdMillis\":1600000000000,\"startedAt\":1600001000000,\"completedAt\":1600001000000," + - "\"location\":\"s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/\"," + - "\"configId\":\"default/container.1\",\"artifacts\":[\"config-dump\"],\"dumpOptions\":{}}"; - assertReportEquals(nodeRepository, expectedJson); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/config-dump.tar.zst")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void handles_multiple_artifact_types() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - when(operations.executeCommandInContainer( - any(NodeAgentContextImpl.class), any(UnixUser.class), any(String[].class))) - // For perf report: - .thenReturn(new CommandResult(null, 0, "12345")) - .thenReturn(new CommandResult(null, 0, "")) - .thenReturn(new CommandResult(null, 0, "")) - // For jfr recording: - .thenReturn(new CommandResult(null, 0, "12345")) - .thenReturn(new CommandResult(null, 0, "ok")) - .thenReturn(new CommandResult(null, 0, "name=host-admin success")); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report", "jvm-jfr"), - new ServiceDumpReport.DumpOptions(true, 20.0, null)); - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - - List<URI> expectedUris = List.of( - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-record.bin.zst"), - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/perf-report.txt"), - URI.create("s3://uri-1/tenant1/service-dump/default-container-1-1600000000000/recording.jfr.zst")); - assertSyncedFiles(context, syncClient, expectedUris); - } - - @Test - void fails_gracefully_on_invalid_request_json() { - // Setup mocks - ContainerOperations operations = mock(ContainerOperations.class); - SyncClient syncClient = createSyncClientMock(); - NodeRepoMock nodeRepository = new NodeRepoMock(); - TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); - JsonNodeFactory fac = new ObjectMapper().getNodeFactory(); - ObjectNode invalidRequest = new ObjectNode(fac) - .set("dumpOptions", new ObjectNode(fac).put("duration", "invalidDurationDataType")); - NodeSpec spec = NodeSpec.Builder - .testSpec(HOSTNAME, NodeState.active) - .report(ServiceDumpReport.REPORT_ID, invalidRequest) - .build(); - nodeRepository.updateNodeSpec(spec); - VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); - NodeAgentContextImpl context = NodeAgentContextImpl.builder(spec) - .fileSystem(fileSystem) - .build(); - reporter.processServiceDumpRequest(context); - String expectedJson = "{\"createdMillis\":1600001000000,\"startedAt\":1600001000000,\"failedAt\":1600001000000," + - "\"configId\":\"unknown\",\"error\":\"Invalid JSON in service dump request\",\"artifacts\":[]}"; - assertReportEquals(nodeRepository, expectedJson); - } - - private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts) { - return createNodeSpecWithDumpRequest(repository, artifacts, new DumpOptions(null, null, null)); - } - - private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts, DumpOptions options) { - ServiceDumpReport request = ServiceDumpReport.createRequestReport( - Instant.ofEpochMilli(1600000000000L), null, "default/container.1", artifacts, options); - NodeSpec spec = NodeSpec.Builder - .testSpec(HOSTNAME, NodeState.active) - .report(ServiceDumpReport.REPORT_ID, request.toJsonNode()) - .archiveUri(URI.create("s3://uri-1/tenant1/")) - .build(); - repository.updateNodeSpec(spec); - return spec; - } - - private static void assertReportEquals(NodeRepoMock nodeRepository, String expectedJson) { - ServiceDumpReport report = nodeRepository.getNode(HOSTNAME).reports() - .getReport(ServiceDumpReport.REPORT_ID, ServiceDumpReport.class).get(); - String actualJson = report.toJson(); - assertEquals(expectedJson, actualJson); - } - - @SuppressWarnings("unchecked") - private static void assertSyncedFiles(NodeAgentContextImpl context, SyncClient client, List<URI> expectedDestinations) { - ArgumentCaptor<List<SyncFileInfo>> filesCaptor = ArgumentCaptor.forClass(List.class); - verify(client).sync(eq(context), filesCaptor.capture(), eq(Integer.MAX_VALUE)); - List<SyncFileInfo> actualFiles = filesCaptor.getValue(); - List<URI> actualFilenames = actualFiles.stream() - .map(SyncFileInfo::destination) - .sorted() - .toList(); - assertEquals(expectedDestinations, actualFilenames); - } - - private SyncClient createSyncClientMock() { - SyncClient client = mock(SyncClient.class); - when(client.sync(any(TaskContext.class), anyList(), anyInt())) - .thenReturn(true); - return client; - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java deleted file mode 100644 index 8e56741274e..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.sync; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.net.URI; -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE; -import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.ZSTD; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class SyncFileInfoTest { - - private static final FileSystem fileSystem = TestFileSystem.create(); - - private static final URI nodeArchiveUri = URI.create("s3://vespa-data-bucket/vespa/music/main/h432a/"); - private static final Path accessLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210211"); - private static final Path accessLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210212.zst"); - private static final Path accessLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/access-json.log.20210213.zst"); - private static final Path accessLogPath4 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.20210214.zst"); - private static final Path accessLogPath5 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.container.20210214.zst"); - private static final Path accessLogPath6 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.metrics-proxy.20210214.zst"); - private static final Path connectionLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210210"); - private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210212.zst"); - private static final Path connectionLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.metrics-proxy.20210210"); - private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log"); - private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); - private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log"); - private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log"); - private static final Path startServicesPath1 = fileSystem.getPath("/opt/vespa/logs/start-services.out"); - private static final Path startServicesPath2 = fileSystem.getPath("/opt/vespa/logs/start-services.out-20230808100143"); - private static final Path rotatedNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log.20231019-1234555"); - private static final Path currentNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log"); - private static final Path nginxAccessLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-access.log.20231019-1234"); - - @Test - void access_logs() { - assertForLogFile(accessLogPath1, null, null, true); - assertForLogFile(accessLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210211.zst", ZSTD, false); - - assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE, true); - assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE, false); - - assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, true); - assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, false); - - assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, true); - assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, false); - - assertForLogFile(accessLogPath5, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.container.20210214.zst", NONE, true); - assertForLogFile(accessLogPath5, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.container.20210214.zst", NONE, false); - - assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, accessLogPath6, true, ApplicationId.defaultId())); - assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, accessLogPath6, false, ApplicationId.defaultId())); - } - - @Test - void connection_logs() { - assertForLogFile(connectionLogPath1, null, null, true); - assertForLogFile(connectionLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210210.zst", ZSTD, false); - - assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, true); - assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, false); - - assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, connectionLogPath3, true, ApplicationId.defaultId())); - assertEquals(Optional.empty(), SyncFileInfo.forLogFile(nodeArchiveUri, connectionLogPath3, false, ApplicationId.defaultId())); - } - - @Test - void vespa_logs() { - new UnixPath(vespaLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true); - assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2022-05-09.14-22-11.zst", ZSTD, Duration.ZERO, false); - - assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true); - assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false); - } - - @Test - void zookeeper_logs() { - new UnixPath(zkLogPath0).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-13T13:13:45Z")); - assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true); - assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-13.13-13-45.zst", ZSTD, Duration.ZERO, false); - - new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true); - assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, false); - } - - @Test - void nginx_error_logs() { - new UnixPath(currentNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ofHours(1),true); - assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ZERO,false); - - new UnixPath(rotatedNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, true); - assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, false); - - // Does not sync access logs - new UnixPath(nginxAccessLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); - Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, nginxAccessLog, false, ApplicationId.defaultId()); - assertEquals(Optional.empty(), sfi); - } - - @Test - void start_services() { - assertForLogFile(startServicesPath1, null, null, true); - assertForLogFile(startServicesPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/start-services/start-services.out-20230808100143.zst", ZSTD, true); - } - - private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) { - assertForLogFile(srcPath, destination, compression, null, rotatedOnly); - } - - private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, Duration minDurationBetweenSync, boolean rotatedOnly) { - Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, srcPath, rotatedOnly, ApplicationId.defaultId()); - assertEquals(destination, sfi.map(SyncFileInfo::destination).map(URI::toString).orElse(null)); - assertEquals(compression, sfi.map(SyncFileInfo::uploadCompression).orElse(null)); - assertEquals(minDurationBetweenSync, sfi.flatMap(SyncFileInfo::minDurationBetweenSync).orElse(null)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java deleted file mode 100644 index 616100363e9..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance.sync; - -import com.yahoo.compress.ZstdCompressor; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Random; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class ZstdCompressingInputStreamTest { - - @Test - void compression_test() { - Random rnd = new Random(); - byte[] data = new byte[(int) (100_000 * (10 + rnd.nextDouble()))]; - rnd.nextBytes(data); - assertCompression(data, 1 << 14); - } - - @Test - void compress_empty_file_test() { - byte[] compressedData = compress(new byte[0], 1 << 10); - assertEquals(13, compressedData.length, "zstd compressing an empty file results in a 13 bytes file"); - } - - private static void assertCompression(byte[] data, int bufferSize) { - byte[] compressedData = compress(data, bufferSize); - byte[] decompressedData = new byte[data.length]; - var compressor = new ZstdCompressor(); - compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); - - assertArrayEquals(data, decompressedData); - } - - private static byte[] compress(byte[] data, int bufferSize) { - ByteArrayInputStream bais = new ByteArrayInputStream(data); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(bais, bufferSize)) { - byte[] buffer = new byte[bufferSize]; - for (int nRead; (nRead = zcis.read(buffer, 0, buffer.length)) != -1; ) - baos.write(buffer, 0, nRead); - baos.flush(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - return baos.toByteArray(); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java deleted file mode 100644 index 355c997a3e0..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.mockito.InOrder; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithScheduler; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithSchedulerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -/** - * @author bakksjo - */ -public class NodeAdminImplTest { - - private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory = mock(NodeAgentWithSchedulerFactory.class); - private final TestTimer timer = new TestTimer(); - private final ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class); - private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, - new Metrics(), timer, Duration.ZERO, Duration.ZERO, procMeminfoReader); - - @Test - void nodeAgentsAreProperlyLifeCycleManaged() { - final NodeAgentContext context1 = createNodeAgentContext("host1.test.yahoo.com"); - final NodeAgentContext context2 = createNodeAgentContext("host2.test.yahoo.com"); - final NodeAgentWithScheduler nodeAgent1 = mockNodeAgentWithSchedulerFactory(context1); - final NodeAgentWithScheduler nodeAgent2 = mockNodeAgentWithSchedulerFactory(context2); - - final InOrder inOrder = inOrder(nodeAgentWithSchedulerFactory, nodeAgent1, nodeAgent2); - nodeAdmin.refreshContainersToRun(Set.of()); - verifyNoMoreInteractions(nodeAgentWithSchedulerFactory); - - nodeAdmin.refreshContainersToRun(Set.of(context1)); - inOrder.verify(nodeAgent1).start(); - inOrder.verify(nodeAgent2, never()).start(); - inOrder.verify(nodeAgent1, never()).stopForRemoval(); - - nodeAdmin.refreshContainersToRun(Set.of(context1)); - inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); - inOrder.verify(nodeAgent1, never()).start(); - inOrder.verify(nodeAgent1, never()).stopForRemoval(); - - nodeAdmin.refreshContainersToRun(Set.of()); - inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); - verify(nodeAgent1).stopForRemoval(); - - nodeAdmin.refreshContainersToRun(Set.of(context2)); - inOrder.verify(nodeAgent2).start(); - inOrder.verify(nodeAgent2, never()).stopForRemoval(); - inOrder.verify(nodeAgent1, never()).stopForRemoval(); - - nodeAdmin.refreshContainersToRun(Set.of()); - inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); - inOrder.verify(nodeAgent2, never()).start(); - inOrder.verify(nodeAgent2).stopForRemoval(); - inOrder.verify(nodeAgent1, never()).start(); - inOrder.verify(nodeAgent1, never()).stopForRemoval(); - } - - @Test - void testSetFrozen() { - Set<NodeAgentContext> contexts = new HashSet<>(); - List<NodeAgentWithScheduler> nodeAgents = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - NodeAgentContext context = createNodeAgentContext("host" + i + ".test.yahoo.com"); - NodeAgentWithScheduler nodeAgent = mockNodeAgentWithSchedulerFactory(context); - - contexts.add(context); - nodeAgents.add(nodeAgent); - } - - nodeAdmin.refreshContainersToRun(contexts); - - assertTrue(nodeAdmin.isFrozen()); // Initially everything is frozen to force convergence - mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); - assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything - - - mockNodeAgentSetFrozenResponse(nodeAgents, false, false, false); - assertFalse(nodeAdmin.setFrozen(true)); // NodeAdmin freezes only when all the NodeAgents are frozen - - mockNodeAgentSetFrozenResponse(nodeAgents, false, true, true); - assertFalse(nodeAdmin.setFrozen(true)); - assertFalse(nodeAdmin.isFrozen()); - - mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); - assertTrue(nodeAdmin.setFrozen(true)); - assertTrue(nodeAdmin.isFrozen()); - - mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); - assertTrue(nodeAdmin.setFrozen(true)); - assertTrue(nodeAdmin.isFrozen()); - - mockNodeAgentSetFrozenResponse(nodeAgents, false, false, false); - assertFalse(nodeAdmin.setFrozen(false)); - assertFalse(nodeAdmin.isFrozen()); // NodeAdmin unfreezes instantly - - mockNodeAgentSetFrozenResponse(nodeAgents, false, false, true); - assertFalse(nodeAdmin.setFrozen(false)); - assertFalse(nodeAdmin.isFrozen()); - - mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); - assertTrue(nodeAdmin.setFrozen(false)); - assertFalse(nodeAdmin.isFrozen()); - } - - @Test - void testSubsystemFreezeDuration() { - // Initially everything is frozen to force convergence - assertTrue(nodeAdmin.isFrozen()); - assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - timer.advance(Duration.ofSeconds(1)); - assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); - - // Unfreezing floors freeze duration - assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything - assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - timer.advance(Duration.ofSeconds(1)); - assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - - // Advancing time now will make freeze duration proceed according to clock - assertTrue(nodeAdmin.setFrozen(true)); - assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - timer.advance(Duration.ofSeconds(1)); - assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); - } - - private void mockNodeAgentSetFrozenResponse(List<NodeAgentWithScheduler> nodeAgents, boolean... responses) { - for (int i = 0; i < nodeAgents.size(); i++) { - NodeAgentWithScheduler nodeAgent = nodeAgents.get(i); - when(nodeAgent.setFrozen(anyBoolean(), any())).thenReturn(responses[i]); - } - } - - private NodeAgentContext createNodeAgentContext(String hostname) { - return NodeAgentContextImpl.builder(hostname).fileSystem(TestFileSystem.create()).build(); - } - - private NodeAgentWithScheduler mockNodeAgentWithSchedulerFactory(NodeAgentContext context) { - NodeAgentWithScheduler nodeAgentWithScheduler = mock(NodeAgentWithScheduler.class); - when(nodeAgentWithSchedulerFactory.create(eq(context))).thenReturn(nodeAgentWithScheduler); - return nodeAgentWithScheduler; - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java deleted file mode 100644 index 420146b52f0..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; - -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.integration.NodeRepoMock; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; -import org.junit.jupiter.api.Test; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.RESUMED; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED; -import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Basic test of NodeAdminStateUpdater - * - * @author freva - */ -public class NodeAdminStateUpdaterTest { - private final NodeAgentContextFactory nodeAgentContextFactory = mock(NodeAgentContextFactory.class); - private final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); - private final Orchestrator orchestrator = mock(Orchestrator.class); - private final NodeAdmin nodeAdmin = mock(NodeAdmin.class); - private final HostName hostHostname = HostName.of("basehost1.test.yahoo.com"); - - private final NodeAdminStateUpdater updater = spy(new NodeAdminStateUpdater( - nodeAgentContextFactory, nodeRepository, orchestrator, nodeAdmin, hostHostname)); - - - @Test - void state_convergence() { - mockNodeRepo(NodeState.active, 4); - List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() - .map(NodeSpec::hostname) - .toList(); - List<String> suspendHostnames = new ArrayList<>(activeHostnames); - suspendHostnames.add(hostHostname.value()); - when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); - - { - // Initially everything is frozen to force convergence - assertConvergeError(RESUMED, "NodeAdmin is not yet unfrozen"); - when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); - updater.converge(RESUMED); - verify(orchestrator, times(1)).resume(hostHostname.value()); - - // We are already resumed, so this should return without resuming again - updater.converge(RESUMED); - verify(orchestrator, times(1)).resume(hostHostname.value()); - verify(nodeAdmin, times(2)).setFrozen(eq(false)); - - // Host is externally suspended in orchestrator, should be resumed by node-admin - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); - updater.converge(RESUMED); - verify(orchestrator, times(2)).resume(hostHostname.value()); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.NO_REMARKS); - - // Lets try to suspend node admin only - when(nodeAdmin.setFrozen(eq(true))).thenReturn(false); - assertConvergeError(SUSPENDED_NODE_ADMIN, "NodeAdmin is not yet frozen"); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - } - - { - // First orchestration failure happens within the freeze convergence timeout, - // and so should not call setFrozen(false) - final String exceptionMessage = "Cannot allow to suspend because some reason"; - when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); - doThrow(new RuntimeException(exceptionMessage)).doNothing() - .when(orchestrator).suspend(eq(hostHostname.value())); - assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMessage); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - - updater.converge(SUSPENDED_NODE_ADMIN); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - verify(orchestrator, times(2)).suspend(hostHostname.value()); - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); - - // Already suspended, no changes - updater.converge(SUSPENDED_NODE_ADMIN); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - verify(orchestrator, times(2)).suspend(hostHostname.value()); - - // Host is externally resumed - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.NO_REMARKS); - updater.converge(SUSPENDED_NODE_ADMIN); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - verify(orchestrator, times(3)).suspend(hostHostname.value()); - setHostOrchestratorStatus(hostHostname, OrchestratorStatus.ALLOWED_TO_BE_DOWN); - } - - { - // At this point orchestrator will say its OK to suspend, but something goes wrong when we try to stop services - final String exceptionMessage = "Failed to stop services"; - verify(orchestrator, times(0)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); - doThrow(new RuntimeException(exceptionMessage)).doNothing().when(nodeAdmin).stopNodeAgentServices(); - assertConvergeError(SUSPENDED, exceptionMessage); - verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); - // Make sure we dont roll back if we fail to stop services - we will try to stop again next tick - verify(nodeAdmin, times(3)).setFrozen(eq(false)); - - // Finally we are successful in transitioning to frozen - updater.converge(SUSPENDED); - } - } - - @Test - void half_transition_revert() { - final String exceptionMsg = "Cannot allow to suspend because some reason"; - mockNodeRepo(NodeState.active, 3); - - // Initially everything is frozen to force convergence - when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); - updater.converge(RESUMED); - verify(nodeAdmin, times(1)).setFrozen(eq(false)); - verify(nodeAdmin, times(1)).refreshContainersToRun(any()); - - // Let's start suspending, we are able to freeze the nodes, but orchestrator denies suspension - when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); - when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); - doThrow(new RuntimeException(exceptionMsg)).when(orchestrator).suspend(eq(hostHostname.value())); - - assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); - verify(nodeAdmin, times(1)).setFrozen(eq(true)); - verify(orchestrator, times(1)).suspend(eq(hostHostname.value())); - assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); - verify(nodeAdmin, times(2)).setFrozen(eq(true)); - verify(orchestrator, times(2)).suspend(eq(hostHostname.value())); - assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); - verify(nodeAdmin, times(3)).setFrozen(eq(true)); - verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); - - // No new unfreezes nor refresh while trying to freeze - verify(nodeAdmin, times(1)).setFrozen(eq(false)); - verify(nodeAdmin, times(1)).refreshContainersToRun(any()); - - // Only resume and fetch containers when subsystem freeze duration expires - when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1)); - assertConvergeError(SUSPENDED_NODE_ADMIN, "Timed out trying to freeze all nodes: will force an unfrozen tick"); - verify(nodeAdmin, times(2)).setFrozen(eq(false)); - verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); // no new suspend calls - verify(nodeAdmin, times(2)).refreshContainersToRun(any()); - - // We change our mind, want to remain resumed - updater.converge(RESUMED); - verify(nodeAdmin, times(3)).setFrozen(eq(false)); // Make sure that we unfreeze! - } - - @Test - void do_not_orchestrate_host_when_not_active() { - when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1)); - when(nodeAdmin.setFrozen(anyBoolean())).thenReturn(true); - mockNodeRepo(NodeState.ready, 3); - - // Resume and suspend only require that node-agents are frozen and permission from - // orchestrator to resume/suspend host. Therefore, if host is not active, we only need to freeze. - updater.converge(RESUMED); - verify(orchestrator, never()).resume(eq(hostHostname.value())); - - updater.converge(SUSPENDED_NODE_ADMIN); - verify(orchestrator, never()).suspend(eq(hostHostname.value())); - - // When doing batch suspend, only suspend the containers if the host is not active - List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() - .map(NodeSpec::hostname) - .toList(); - updater.converge(SUSPENDED); - verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(activeHostnames)); - } - - @Test - void node_spec_and_acl_aligned() { - Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(NodeState.active, 3); - mockAcl(acl, 1, 2, 3); - - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(acl)); - verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); - verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); - } - - @Test - void node_spec_and_acl_mismatch_missing_one_acl() { - Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(NodeState.active, 3); - mockAcl(acl, 1, 2); // Acl for 3 is missing - - updater.adjustNodeAgentsToRunFromNodeRepository(); - mockNodeRepo(NodeState.active, 2); // Next tick, the spec for 3 is no longer returned - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(1)).create(argThat(spec -> spec.hostname().equals("host3.yahoo.com")), eq(Acl.EMPTY)); - verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); - verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); - } - - @Test - void node_spec_and_acl_mismatch_additional_acl() { - Acl acl = new Acl.Builder().withTrustedPorts(22).build(); - mockNodeRepo(NodeState.active, 2); - mockAcl(acl, 1, 2, 3); // Acl for 3 is extra - - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - updater.adjustNodeAgentsToRunFromNodeRepository(); - - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host1.yahoo.com")), eq(acl)); - verify(nodeAgentContextFactory, times(3)).create(argThat(spec -> spec.hostname().equals("host2.yahoo.com")), eq(acl)); - verify(nodeRepository, times(3)).getNodes(eq(hostHostname.value())); - verify(nodeRepository, times(3)).getAcls(eq(hostHostname.value())); - } - - private void assertConvergeError(NodeAdminStateUpdater.State targetState, String reason) { - try { - updater.converge(targetState); - fail("Expected converging to " + targetState + " to fail with \"" + reason + "\", but it succeeded without error"); - } catch (RuntimeException e) { - assertEquals(reason, e.getMessage()); - } - } - - private void mockNodeRepo(NodeState hostState, int numberOfNodes) { - nodeRepository.resetNodeSpecs(); - - IntStream.rangeClosed(1, numberOfNodes) - .mapToObj(i -> NodeSpec.Builder.testSpec("host" + i + ".yahoo.com").parentHostname(hostHostname.value()).build()) - .forEach(nodeRepository::updateNodeSpec); - - nodeRepository.updateNodeSpec(NodeSpec.Builder.testSpec(hostHostname.value(), hostState).type(NodeType.host).build()); - } - - private void mockAcl(Acl acl, int... nodeIds) { - nodeRepository.setAcl(Arrays.stream(nodeIds) - .mapToObj(i -> "host" + i + ".yahoo.com") - .collect(Collectors.toMap(Function.identity(), h -> acl))); - } - - private void setHostOrchestratorStatus(HostName hostname, OrchestratorStatus orchestratorStatus) { - nodeRepository.updateNodeSpec(hostname.value(), node -> node.orchestratorStatus(orchestratorStatus)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java deleted file mode 100644 index 589eceebb74..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author freva - */ -public class NodeAgentContextImplTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final NodeAgentContext context = NodeAgentContextImpl.builder("container-1.domain.tld") - .fileSystem(fileSystem).build(); - - @Test - void path_on_host_from_path_in_node_test() { - assertEquals( - "/data/vespa/storage/container-1", - context.paths().of("/").pathOnHost().toString()); - - assertEquals( - "/data/vespa/storage/container-1/dev/null", - context.paths().of("/dev/null").pathOnHost().toString()); - } - - @Test - void path_in_container_must_be_absolute() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().of("some/relative/path"); - }); - } - - @Test - void path_in_node_from_path_on_host_test() { - assertEquals( - "/dev/null", - context.paths().fromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-1/dev/null")).pathInContainer()); - } - - @Test - void path_on_host_must_be_absolute() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().fromPathOnHost(Path.of("some/relative/path")); - }); - } - - @Test - void path_on_host_must_be_inside_container_storage_of_context() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().fromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-2/dev/null")); - }); - } - - @Test - void path_on_host_must_be_inside_container_storage() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().fromPathOnHost(fileSystem.getPath("/home")); - }); - } - - @Test - void path_under_vespa_host_in_container_test() { - assertEquals( - "/opt/vespa", - context.paths().underVespaHome("").pathInContainer()); - - assertEquals( - "/opt/vespa/logs/vespa/vespa.log", - context.paths().underVespaHome("logs/vespa/vespa.log").pathInContainer()); - } - - @Test - void path_under_vespa_home_must_be_relative() { - assertThrows(IllegalArgumentException.class, () -> { - context.paths().underVespaHome("/home"); - }); - } - - @Test - void disabledTasksTest() { - NodeAgentContext context1 = createContextWithDisabledTasks(); - assertFalse(context1.isDisabled(NodeAgentTask.DiskCleanup)); - assertFalse(context1.isDisabled(NodeAgentTask.CoreDumps)); - - NodeAgentContext context2 = createContextWithDisabledTasks("root>UpgradeTask", "DiskCleanup", "node>CoreDumps"); - assertFalse(context2.isDisabled(NodeAgentTask.DiskCleanup)); - assertTrue(context2.isDisabled(NodeAgentTask.CoreDumps)); - } - - private NodeAgentContext createContextWithDisabledTasks(String... tasks) { - InMemoryFlagSource flagSource = new InMemoryFlagSource(); - flagSource.withListFlag(PermanentFlags.DISABLED_HOST_ADMIN_TASKS.id(), List.of(tasks), String.class); - return NodeAgentContextImpl.builder("node123").fileSystem(fileSystem).flagSource(flagSource).build(); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java deleted file mode 100644 index 5e09c45d217..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.jdisc.core.SystemTimer; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; - -import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextSupplier.ContextSupplierInterruptedException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - */ -public class NodeAgentContextManagerTest { - - private static final int TIMEOUT = 10_000; - - private final SystemTimer timer = new SystemTimer(); - private final NodeAgentContext initialContext = generateContext(); - private final NodeAgentContextManager manager = new NodeAgentContextManager(timer, initialContext); - - @Test - @Timeout(TIMEOUT) - void context_is_ignored_unless_scheduled_while_waiting() { - NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1, timer.currentTime()); - assertSame(initialContext, manager.currentContext()); - - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - assertFalse(async.isCompleted()); - - NodeAgentContext context2 = generateContext(); - manager.scheduleTickWith(context2, timer.currentTime()); - - assertSame(context2, async.awaitResult().response.get()); - assertSame(context2, manager.currentContext()); - } - - @Test - @Timeout(TIMEOUT) - void returns_no_earlier_than_at_given_time() { - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - - NodeAgentContext context1 = generateContext(); - Instant returnAt = timer.currentTime().plusMillis(500); - manager.scheduleTickWith(context1, returnAt); - - assertSame(context1, async.awaitResult().response.get()); - assertSame(context1, manager.currentContext()); - // Is accurate to a millisecond - assertFalse(timer.currentTime().plusMillis(1).isBefore(returnAt)); - } - - @Test - @Timeout(TIMEOUT) - void blocks_in_nextContext_until_one_is_scheduled() { - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - assertFalse(async.isCompleted()); - - NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1, timer.currentTime()); - - async.awaitResult(); - assertEquals(Optional.of(context1), async.response); - assertFalse(async.exception.isPresent()); - } - - @Test - @Timeout(TIMEOUT) - void blocks_in_nextContext_until_interrupt() { - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - assertFalse(async.isCompleted()); - - manager.interrupt(); - - async.awaitResult(); - assertEquals(Optional.of(ContextSupplierInterruptedException.class), async.exception.map(Exception::getClass)); - assertFalse(async.response.isPresent()); - } - - @Test - @Timeout(TIMEOUT) - void setFrozen_does_not_block_with_no_timeout() { - assertFalse(manager.setFrozen(false, Duration.ZERO)); - - // Generate new context and get it from the supplier, this completes the unfreeze - NodeAgentContext context1 = generateContext(); - AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - manager.waitUntilWaitingForNextContext(); - manager.scheduleTickWith(context1, timer.currentTime()); - assertSame(context1, async.awaitResult().response.get()); - - assertTrue(manager.setFrozen(false, Duration.ZERO)); - } - - @Test - @Timeout(TIMEOUT) - void setFrozen_blocks_at_least_for_duration_of_timeout() { - long wantedDurationMillis = 100; - long start = timer.currentTimeMillis(); - assertFalse(manager.setFrozen(false, Duration.ofMillis(wantedDurationMillis))); - long actualDurationMillis = timer.currentTimeMillis() - start; - - assertTrue(actualDurationMillis >= wantedDurationMillis); - } - - private static NodeAgentContext generateContext() { - return NodeAgentContextImpl.builder("container-123.domain.tld").fileSystem(TestFileSystem.create()).build(); - } - - private static class AsyncExecutor<T> { - private final CountDownLatch latch = new CountDownLatch(1); - private volatile Optional<T> response = Optional.empty(); - private volatile Optional<Exception> exception = Optional.empty(); - - private AsyncExecutor(Callable<T> supplier) { - new Thread(() -> { - try { - response = Optional.of(supplier.call()); - } catch (Exception e) { - exception = Optional.of(e); - } - latch.countDown(); - }).start(); - } - - private AsyncExecutor<T> awaitResult() { - try { - latch.await(); - } catch (InterruptedException ignored) { } - return this; - } - - private boolean isCompleted() { - return latch.getCount() == 0; - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java deleted file mode 100644 index 709326cc3b8..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ /dev/null @@ -1,889 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; -import com.yahoo.vespa.hosted.node.admin.container.Container; -import com.yahoo.vespa.hosted.node.admin.container.ContainerId; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; -import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; -import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.InOrder; - -import java.nio.file.FileSystem; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.BiFunction; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author Øyvind Bakksjø - */ -public class NodeAgentImplTest { - private static final NodeResources resources = new NodeResources(2, 16, 250, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local); - private static final Version vespaVersion = Version.fromString("1.2.3"); - private static final ContainerId containerId = new ContainerId("af23"); - private static final String hostName = "host1.test.yahoo.com"; - - private final NodeAgentContextSupplier contextSupplier = mock(NodeAgentContextSupplier.class); - private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); - private final ContainerOperations containerOperations = mock(ContainerOperations.class); - private final NodeRepository nodeRepository = mock(NodeRepository.class); - private final Orchestrator orchestrator = mock(Orchestrator.class); - private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); - private final HealthChecker healthChecker = mock(HealthChecker.class); - private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private final TestTimer timer = new TestTimer(Instant.now()); - private final FileSystem fileSystem = TestFileSystem.create(); - - @BeforeEach - public void setUp() { - when(containerOperations.suspendNode(any())).thenReturn(""); - when(containerOperations.resumeNode(any())).thenReturn(""); - when(containerOperations.restartVespa(any())).thenReturn(""); - when(containerOperations.startServices(any())).thenReturn(""); - when(containerOperations.stopServices(any())).thenReturn(""); - } - - @Test - void upToDateContainerIsUntouched() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .orchestratorStatus(OrchestratorStatus.NO_REMARKS) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(orchestrator, never()).suspend(any(String.class)); - verify(containerOperations, never()).pullImageAsyncIfNeeded(any(), any(), any()); - - final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository); - // TODO: Verify this isn't run unless 1st time - inOrder.verify(containerOperations, never()).startServices(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - inOrder.verify(orchestrator, never()).resume(hostName); - } - - @Test - void verifyRemoveOldFilesIfDiskFull() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(storageMaintainer, times(1)).cleanDiskIfFull(eq(context)); - } - - @Test - void startsAfterStoppingServices() { - final InOrder inOrder = inOrder(containerOperations); - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - inOrder.verify(containerOperations, never()).startServices(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - - nodeAgent.stopForHostSuspension(context); - nodeAgent.doConverge(context); - inOrder.verify(containerOperations, never()).startServices(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services - - // No new suspends/stops, so no need to resume/start - nodeAgent.doConverge(context); - inOrder.verify(containerOperations, never()).startServices(eq(context)); - inOrder.verify(containerOperations, never()).resumeNode(eq(context)); - - nodeAgent.stopForHostSuspension(context); - nodeAgent.doConverge(context); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); - inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); - inOrder.verify(containerOperations, times(0)).startServices(eq(context)); // done as part of startContainer - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - } - - @Test - void absentContainerCausesStart() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, never()).startServices(any()); - verify(orchestrator, never()).suspend(any(String.class)); - - final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer, healthChecker); - inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any()); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); - inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); - inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context)); - inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0)); - inOrder.verify(orchestrator, never()).resume(hostName); - } - - @Test - void containerIsNotStoppedIfNewImageMustBePulled() { - final DockerImage newDockerImage = DockerImage.fromString("registry.example.com/repo/new-image"); - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(newDockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); - - nodeAgent.doConverge(context); - - verify(orchestrator, never()).suspend(any(String.class)); - verify(orchestrator, never()).resume(any(String.class)); - verify(containerOperations, never()).removeContainer(eq(context), any()); - - final InOrder inOrder = inOrder(containerOperations); - inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(newDockerImage), any()); - } - - @Test - void containerIsUpdatedIfCpuChanged() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .orchestratorStatus(OrchestratorStatus.NO_REMARKS); - - NodeAgentContext firstContext = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); - - InOrder inOrder = inOrder(orchestrator, containerOperations); - - nodeAgent.doConverge(firstContext); - inOrder.verify(orchestrator, never()).resume(any(String.class)); - - NodeAgentContext secondContext = createContext(specBuilder.diskGb(200).build()); - nodeAgent.doConverge(secondContext); - inOrder.verify(orchestrator, never()).resume(any(String.class)); - - NodeAgentContext thirdContext = NodeAgentContextImpl.builder(specBuilder.vcpu(5).build()).fileSystem(fileSystem).cpuSpeedUp(1.25).build(); - nodeAgent.doConverge(thirdContext); - ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16); - mockGetContainer(dockerImage, resourcesAfterThird, true); - - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(containerId), eq(resourcesAfterThird)); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(containerOperations, never()).startContainer(any()); - inOrder.verify(orchestrator, never()).resume(any()); - - // No changes - nodeAgent.converge(thirdContext); - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations, never()).updateContainer(eq(thirdContext), eq(containerId), any()); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator, never()).resume(any()); - - // Set the feature flag - flagSource.withDoubleFlag(PermanentFlags.CONTAINER_CPU_CAP.id(), 2.3); - - nodeAgent.doConverge(thirdContext); - inOrder.verify(containerOperations).updateContainer(eq(thirdContext), eq(containerId), eq(ContainerResources.from(9.2, 4, 16))); - inOrder.verify(orchestrator, never()).resume(any()); - } - - @Test - void containerIsRecreatedIfMemoryChanged() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(2).currentRestartGeneration(1); - - NodeAgentContext firstContext = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(containerOperations.pullImageAsyncIfNeeded(any(), any(), any())).thenReturn(true); - - nodeAgent.doConverge(firstContext); - NodeAgentContext secondContext = createContext(specBuilder.memoryGb(20).build()); - nodeAgent.doConverge(secondContext); - ContainerResources resourcesAfterThird = ContainerResources.from(0, 2, 20); - mockGetContainer(dockerImage, resourcesAfterThird, true); - - InOrder inOrder = inOrder(orchestrator, containerOperations, nodeRepository); - inOrder.verify(orchestrator).resume(any(String.class)); - inOrder.verify(containerOperations).removeContainer(eq(secondContext), any()); - inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - inOrder.verify(containerOperations, never()).restartVespa(any()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2).withRebootGeneration(0))); - - nodeAgent.doConverge(secondContext); - inOrder.verify(orchestrator).resume(any(String.class)); - inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - } - - @Test - void noRestartIfOrchestratorSuspendFails() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(2).currentRestartGeneration(1) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - doThrow(new OrchestratorException("Denied")).when(orchestrator).suspend(eq(hostName)); - try { - nodeAgent.doConverge(context); - fail("Expected to throw an exception"); - } catch (OrchestratorException ignored) { - } - - verify(containerOperations, never()).createContainer(eq(context), any()); - verify(containerOperations, never()).startContainer(eq(context)); - verify(orchestrator, never()).resume(any(String.class)); - verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class)); - - // Verify aclMaintainer is called even if suspension fails - verify(aclMaintainer, times(1)).converge(eq(context)); - } - - @Test - void recreatesContainerIfRebootWanted() { - final long wantedRebootGeneration = 2; - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRebootGeneration(wantedRebootGeneration).currentRebootGeneration(1) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); - doThrow(ConvergenceException.ofTransient("Connection refused")).doNothing() - .when(healthChecker).verifyHealth(eq(context)); - - try { - nodeAgent.doConverge(context); - } catch (ConvergenceException ignored) { - } - - // First time we fail to resume because health verification fails - verify(orchestrator, times(1)).suspend(eq(hostName)); - verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any()); - verify(containerOperations, times(1)).startContainer(eq(context)); - verify(orchestrator, never()).resume(eq(hostName)); - verify(nodeRepository, never()).updateNodeAttributes(any(), any()); - - nodeAgent.doConverge(context); - - // Do not reboot the container again - verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any()); - verify(orchestrator, times(1)).resume(eq(hostName)); - verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() - .withRebootGeneration(wantedRebootGeneration))); - } - - @Test - void failedNodeRunningContainerShouldStillBeRunning() { - final NodeSpec node = nodeBuilder(NodeState.failed) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(orchestrator, never()).resume(any(String.class)); - verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); - } - - @Test - void readyNodeLeadsToNoAction() { - final NodeSpec node = nodeBuilder(NodeState.ready).build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - nodeAgent.doConverge(context); - nodeAgent.doConverge(context); - - // Should only be called once, when we initialize - verify(containerOperations, times(1)).getContainer(eq(context)); - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, never()).createContainer(eq(context), any()); - verify(containerOperations, never()).startContainer(eq(context)); - verify(orchestrator, never()).resume(any(String.class)); - verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); - } - - @Test - void inactiveNodeRunningContainerShouldStillBeRunning() { - final NodeSpec node = nodeBuilder(NodeState.inactive) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - final InOrder inOrder = inOrder(storageMaintainer, containerOperations); - inOrder.verify(containerOperations, never()).removeContainer(eq(context), any()); - - verify(orchestrator, never()).resume(any(String.class)); - verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); - } - - @Test - void reservedNodeDoesNotUpdateNodeRepoWithVersion() { - final NodeSpec node = nodeBuilder(NodeState.reserved) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); - } - - private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState nodeState, Optional<Long> wantedRestartGeneration) { - NodeSpec.Builder builder = nodeBuilder(nodeState) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage); - wantedRestartGeneration.ifPresent(restartGeneration -> builder - .wantedRestartGeneration(restartGeneration).currentRestartGeneration(restartGeneration)); - - NodeSpec node = builder.build(); - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - final InOrder inOrder = inOrder(storageMaintainer, containerOperations, nodeRepository); - inOrder.verify(containerOperations, times(1)).stopServices(eq(context)); - inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), any(), eq(true)); - inOrder.verify(containerOperations, times(1)).removeContainer(eq(context), any()); - inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context)); - inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(NodeState.ready)); - - verify(containerOperations, never()).createContainer(eq(context), any()); - verify(containerOperations, never()).startContainer(eq(context)); - verify(containerOperations, never()).suspendNode(eq(context)); - verify(containerOperations, times(1)).stopServices(eq(context)); - verify(orchestrator, never()).resume(any(String.class)); - verify(orchestrator, never()).suspend(any(String.class)); - // current Docker image and vespa version should be cleared - verify(nodeRepository, times(1)).updateNodeAttributes( - eq(hostName), eq(new NodeAttributes().withDockerImage(DockerImage.EMPTY).withVespaVersion(Version.emptyVersion))); - } - - @Test - void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycled() { - nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.of(1L)); - } - - @Test - void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycledNoRestartGeneration() { - nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState.dirty, Optional.empty()); - } - - @Test - void testRestartDeadContainerAfterNodeAdminRestart() { - final NodeSpec node = nodeBuilder(NodeState.active) - .currentDockerImage(dockerImage).wantedDockerImage(dockerImage) - .currentVespaVersion(vespaVersion) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false); - - when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any()); - verify(containerOperations, times(1)).startContainer(eq(context)); - } - - @Test - void resumeProgramRunsUntilSuccess() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .currentVespaVersion(vespaVersion) - .wantedRestartGeneration(1).currentRestartGeneration(1) - .orchestratorStatus(OrchestratorStatus.ALLOWED_TO_BE_DOWN) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - - final InOrder inOrder = inOrder(orchestrator, containerOperations, nodeRepository); - doThrow(new RuntimeException("Failed 1st time")) - .doReturn("") - .when(containerOperations).resumeNode(eq(context)); - - // 1st try - try { - nodeAgent.doConverge(context); - fail("Expected to throw an exception"); - } catch (RuntimeException ignored) { - } - - inOrder.verify(containerOperations, times(1)).resumeNode(any()); - inOrder.verifyNoMoreInteractions(); - - // 2nd try - nodeAgent.doConverge(context); - - inOrder.verify(containerOperations).resumeNode(any()); - inOrder.verify(orchestrator).resume(hostName); - inOrder.verifyNoMoreInteractions(); - } - - @Test - void start_container_subtask_failure_leads_to_container_restart() { - final NodeSpec node = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion) - .wantedRestartGeneration(1).currentRestartGeneration(1) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false)); - - when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); - doThrow(new RuntimeException("Failed to set up network")).doNothing().when(containerOperations).startContainer(eq(context)); - - try { - nodeAgent.doConverge(context); - fail("Expected to get RuntimeException"); - } catch (RuntimeException ignored) { - } - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any()); - verify(containerOperations, times(1)).startContainer(eq(context)); - verify(nodeAgent, never()).resumeNodeIfNeeded(any()); - - // The docker container was actually started and is running, but subsequent exec calls to set up - // networking failed - mockGetContainer(dockerImage, true); - nodeAgent.doConverge(context); - - verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(2)).createContainer(eq(context), any()); - verify(containerOperations, times(2)).startContainer(eq(context)); - verify(nodeAgent, times(1)).resumeNodeIfNeeded(any()); - } - - @Test - void testRunningConfigServer() { - final NodeSpec node = nodeBuilder(NodeState.active) - .type(NodeType.config) - .wantedDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion) - .orchestratorStatus(OrchestratorStatus.ALLOWED_TO_BE_DOWN) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(containerOperations.pullImageAsyncIfNeeded(any(), eq(dockerImage), any())).thenReturn(false); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(orchestrator, never()).suspend(any(String.class)); - - final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer); - inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any()); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); - inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); - inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); - inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); - inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0)); - inOrder.verify(orchestrator).resume(hostName); - } - - - // Tests that only containers without owners are stopped - @Test - void testThatStopContainerDependsOnOwnerPresent() { - verifyThatContainerIsStopped(NodeState.parked, Optional.empty()); - verifyThatContainerIsStopped(NodeState.parked, Optional.of(ApplicationId.defaultId())); - verifyThatContainerIsStopped(NodeState.failed, Optional.empty()); - verifyThatContainerIsStopped(NodeState.failed, Optional.of(ApplicationId.defaultId())); - verifyThatContainerIsStopped(NodeState.inactive, Optional.of(ApplicationId.defaultId())); - } - - @Test - void initial_cpu_cap_test() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion); - - NodeAgentContext context = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30)); - - InOrder inOrder = inOrder(orchestrator, containerOperations); - - ConvergenceException healthCheckException = ConvergenceException.ofTransient("Not yet up"); - doThrow(healthCheckException).when(healthChecker).verifyHealth(any()); - for (int i = 0; i < 3; i++) { - try { - nodeAgent.doConverge(context); - fail("Expected to fail with health check exception"); - } catch (ConvergenceException e) { - assertEquals(healthCheckException, e); - } - timer.advance(Duration.ofSeconds(30)); - } - - doNothing().when(healthChecker).verifyHealth(any()); - try { - nodeAgent.doConverge(context); - fail("Expected to fail due to warm up period not yet done"); - } catch (ConvergenceException e) { - assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); - } - inOrder.verify(orchestrator, never()).resume(any()); - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - - - timer.advance(Duration.ofSeconds(31)); - nodeAgent.doConverge(context); - - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 2, 16))); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(containerOperations, never()).startContainer(any()); - inOrder.verify(orchestrator, never()).resume(any()); - - // No changes - nodeAgent.converge(context); - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations, never()).updateContainer(eq(context), eq(containerId), any()); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator, never()).resume(any()); - } - - @Test - void resumes_normally_if_container_is_already_capped_on_start() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(1).currentRestartGeneration(1); - - NodeAgentContext context = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); - mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); - - InOrder inOrder = inOrder(orchestrator, containerOperations); - - nodeAgent.doConverge(context); - - nodeAgent.converge(context); - inOrder.verify(orchestrator, never()).suspend(any(String.class)); - inOrder.verify(containerOperations, never()).updateContainer(eq(context), eq(containerId), any()); - inOrder.verify(containerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator, never()).resume(any(String.class)); - } - - @Test - void uncaps_and_caps_cpu_for_services_restart() { - NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(2).currentRestartGeneration(1); - - NodeAgentContext context = createContext(specBuilder.build()); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); - mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); - - InOrder inOrder = inOrder(orchestrator, containerOperations); - - nodeAgent.converge(context); - inOrder.verify(orchestrator, times(1)).suspend(eq(hostName)); - inOrder.verify(containerOperations, times(1)).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 0, 16))); - inOrder.verify(containerOperations, times(1)).restartVespa(eq(context)); - - mockGetContainer(dockerImage, ContainerResources.from(0, 0, 16), true); - doNothing().when(healthChecker).verifyHealth(any()); - try { - nodeAgent.doConverge(context); - fail("Expected to fail due to warm up period not yet done"); - } catch (ConvergenceException e) { - assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); - } - inOrder.verify(orchestrator, never()).resume(any()); - inOrder.verify(orchestrator, never()).suspend(any()); - inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - - - timer.advance(Duration.ofSeconds(31)); - nodeAgent.doConverge(context); - inOrder.verify(orchestrator, times(1)).resume(eq(hostName)); - } - - @Test - void resume_during_first_warmup() { - InOrder inOrder = inOrder(orchestrator, nodeRepository); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); - mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); - - // Warmup period prevents resume when node has a current docker image, i.e., already existed. - nodeAgent.converge(createContext(nodeBuilder(NodeState.active).wantedDockerImage(dockerImage).currentDockerImage(dockerImage).build())); - inOrder.verifyNoMoreInteractions(); - - nodeAgent.converge(createContext(nodeBuilder(NodeState.active).wantedDockerImage(dockerImage).build())); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withDockerImage(dockerImage) - .withRebootGeneration(0) - .withVespaVersion(Version.fromString("7.1.1")))); - inOrder.verifyNoMoreInteractions(); - } - - - @Test - void drop_all_documents() { - InOrder inOrder = inOrder(orchestrator, nodeRepository); - BiFunction<NodeState, DropDocumentsReport, NodeSpec> specBuilder = (state, report) -> (report == null ? - nodeBuilder(state) : nodeBuilder(state).report(DropDocumentsReport.reportId(), report.toJsonNode())) - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .build(); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); - - NodeAgentContext context = createContext(specBuilder.apply(NodeState.active, null)); - UnixPath indexPath = new UnixPath(context.paths().underVespaHome("var/db/vespa/search/cluster.foo/0/doc")).createParents().createNewFile(); - mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); - assertTrue(indexPath.exists()); - - // Initially no changes, index is not dropped - nodeAgent.converge(context); - assertTrue(indexPath.exists()); - inOrder.verifyNoMoreInteractions(); - - context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, null, null, null))); - nodeAgent.converge(context); - verify(containerOperations).removeContainer(eq(context), any()); - assertFalse(indexPath.exists()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, timer.currentTimeMillis(), null, null).toJsonNode()))); - inOrder.verifyNoMoreInteractions(); - - // After droppedAt and before readiedAt are set, we cannot proceed - mockGetContainer(null, false); - context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, null, null))); - nodeAgent.converge(context); - verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, never()).startContainer(eq(context)); - inOrder.verifyNoMoreInteractions(); - - context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, 3L, null))); - nodeAgent.converge(context); - verify(containerOperations).startContainer(eq(context)); - inOrder.verifyNoMoreInteractions(); - - mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); - timer.advance(Duration.ofSeconds(31)); - nodeAgent.converge(context); - verify(containerOperations, times(1)).startContainer(eq(context)); - verify(containerOperations, never()).removeContainer(eq(context), any()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() - .withRebootGeneration(0) - .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, timer.currentTimeMillis()).toJsonNode()))); - inOrder.verifyNoMoreInteractions(); - } - - private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { - NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState) - .type(NodeType.tenant) - .flavor("docker") - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage); - - owner.ifPresent(nodeBuilder::owner); - NodeSpec node = nodeBuilder.build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - - when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - - verify(containerOperations, never()).removeContainer(eq(context), any()); - if (owner.isPresent()) { - verify(containerOperations, never()).stopServices(eq(context)); - } else { - verify(containerOperations, times(1)).stopServices(eq(context)); - nodeAgent.doConverge(context); - // Should not be called more than once, have already been stopped - verify(containerOperations, times(1)).stopServices(eq(context)); - } - } - - private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) { - return makeNodeAgent(dockerImage, isRunning, Duration.ofSeconds(-1)); - } - - private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning, Duration warmUpDuration) { - mockGetContainer(dockerImage, isRunning); - doAnswer(invoc -> { - NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); - ContainerResources resources = invoc.getArgument(1, ContainerResources.class); - mockGetContainer(context.node().wantedDockerImage().get(), resources, true); - return null; - }).when(containerOperations).createContainer(any(), any()); - - doAnswer(invoc -> { - NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); - ContainerResources resources = invoc.getArgument(2, ContainerResources.class); - mockGetContainer(context.node().wantedDockerImage().get(), resources, true); - return null; - }).when(containerOperations).updateContainer(any(), any(), any()); - - return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, - () -> RegistryCredentials.none, storageMaintainer, flagSource, - List.of(credentialsMaintainer), Optional.of(aclMaintainer), Optional.of(healthChecker), - timer, warmUpDuration, VespaServiceDumper.DUMMY_INSTANCE, List.of()); - } - - private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { - mockGetContainer(dockerImage, ContainerResources.from(0, resources.vcpu(), resources.memoryGb()), isRunning); - } - - private void mockGetContainer(DockerImage dockerImage, ContainerResources containerResources, boolean isRunning) { - doAnswer(invoc -> { - NodeAgentContext context = invoc.getArgument(0); - if (!hostName.equals(context.hostname().value())) - throw new IllegalArgumentException(); - return dockerImage != null ? - Optional.of(new Container( - containerId, - ContainerName.fromHostname(hostName), - timer.currentTime(), - isRunning ? Container.State.running : Container.State.exited, - "image-id-1", - dockerImage, - Map.of(), - 42, - 43, - hostName, - containerResources, - List.of(), - true)) : - Optional.empty(); - }).when(containerOperations).getContainer(any()); - } - - private NodeAgentContext createContext(NodeSpec nodeSpec) { - return NodeAgentContextImpl.builder(nodeSpec).fileSystem(fileSystem).build(); - } - - private NodeSpec.Builder nodeBuilder(NodeState state) { - return NodeSpec.Builder.testSpec(hostName, state).realResources(resources); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java deleted file mode 100644 index c45d9ab4b61..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespaceTest.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeagent; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author freva - */ -class UserNamespaceTest { - - private final UserNamespace userNamespace = new UserNamespace(1000, 2000, 10000); - - @Test - public void translates_between_ids() { - assertEquals(1001, userNamespace.userIdOnHost(1)); - assertEquals(2001, userNamespace.groupIdOnHost(1)); - assertEquals(1, userNamespace.userIdInContainer(1001)); - assertEquals(1, userNamespace.groupIdInContainer(2001)); - - assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(1)); - assertEquals(userNamespace.overflowId(), userNamespace.userIdInContainer(999999)); - - assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(-1)); - assertThrows(IllegalArgumentException.class, () -> userNamespace.userIdOnHost(70_000)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java deleted file mode 100644 index eddc7edd597..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.provider; - -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class DebugHandlerHelperTest { - @Test - void trivial() { - DebugHandlerHelper helper = new DebugHandlerHelper(); - helper.addConstant("constant-key", "constant-value"); - - NodeAdminDebugHandler handler = () -> Map.of("handler-value-key", "handler-value-value"); - helper.addHandler("handler-key", handler); - - helper.addThreadSafeSupplier("supplier-key", () -> "supplier-value"); - - assertEquals("{" + - "supplier-key=supplier-value, " + - "handler-key={handler-value-key=handler-value-value}, " + - "constant-key=constant-value" + - "}", - helper.getDebugPage().toString()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java deleted file mode 100644 index 115969c5ded..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.logging.Logger; - -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -/** - * @author bjorncs - */ -public class DefaultEnvWriterTest { - - @TempDir - public File temporaryFolder; - - private static final Path EXAMPLE_FILE = Path.of("src/test/resources/default-env-example.txt"); - private static final Path EXPECTED_RESULT_FILE = Path.of("src/test/resources/default-env-rewritten.txt"); - - private final TaskContext context = mock(TaskContext.class); - - @Test - void default_env_is_correctly_rewritten() throws IOException { - Path tempFile = File.createTempFile("junit", null, temporaryFolder).toPath(); - Files.copy(EXAMPLE_FILE, tempFile, REPLACE_EXISTING); - - DefaultEnvWriter writer = new DefaultEnvWriter(); - writer.addOverride("VESPA_HOSTNAME", "my-new-hostname"); - writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver"); - writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file"); - - boolean modified = writer.updateFile(context, tempFile); - - assertTrue(modified); - assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile)); - verify(context, times(1)).log(any(Logger.class), any(String.class)); - - modified = writer.updateFile(context, tempFile); - assertFalse(modified); - assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile)); - verify(context, times(1)).log(any(Logger.class), any(String.class)); - } - - @Test - void generates_default_env_content() throws IOException { - DefaultEnvWriter writer = new DefaultEnvWriter(); - writer.addOverride("VESPA_HOSTNAME", "my-new-hostname"); - writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver"); - writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file"); - writer.addUnset("VESPA_LEGACY_OPTION"); - String generatedContent = writer.generateContent(); - assertEquals(Files.readString(EXPECTED_RESULT_FILE), generatedContent); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java deleted file mode 100644 index 76676739613..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/StringEditorTest.java +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import org.junit.jupiter.api.Test; - -import java.util.Optional; -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.Assertions.*; - -public class StringEditorTest { - private final StringEditor editor = new StringEditor(); - private final Cursor cursor = editor.cursor(); - - @Test - void testBasics() { - assertCursor(0, 0, ""); - - cursor.write("hello"); - assertCursor(0, 5, "hello"); - - cursor.write("one\ntwo"); - assertCursor(1, 3, "helloone\ntwo"); - - cursor.deleteAll(); - assertCursor(0, 0, ""); - - cursor.moveForward(); - assertCursor(0, 0, ""); - - cursor.writeLine("foo"); - assertCursor(1, 0, "foo\n"); - - cursor.writeLines("one", "two"); - assertCursor(3, 0, "foo\none\ntwo\n"); - - cursor.deleteBackward(); - assertCursor(2, 3, "foo\none\ntwo"); - - cursor.deleteBackward(2); - assertCursor(2, 1, "foo\none\nt"); - - Mark mark = cursor.createMark(); - - cursor.moveToStartOfPreviousLine().moveBackward(2); - assertCursor(0, 2, "foo\none\nt"); - - assertEquals("o\none\nt", cursor.getTextTo(mark)); - - cursor.deleteTo(mark); - assertCursor(0, 2, "fo"); - - cursor.deleteBackward(2); - assertCursor(0, 0, ""); - - cursor.writeLines("one", "two", "three").moveToStartOfBuffer(); - assertCursor(0, 0, "one\ntwo\nthree\n"); - - Pattern pattern = Pattern.compile("t(.)"); - Optional<Match> match = cursor.moveForwardToEndOfMatch(pattern); - assertCursor(1, 2, "one\ntwo\nthree\n"); - assertTrue(match.isPresent()); - assertEquals("tw", match.get().match()); - assertEquals("", match.get().prefix()); - assertEquals("o", match.get().suffix()); - assertEquals(new Position(1, 0), match.get().startOfMatch()); - assertEquals(new Position(1, 2), match.get().endOfMatch()); - assertEquals(1, match.get().groupCount()); - assertEquals("w", match.get().group(1)); - - match = cursor.moveForwardToEndOfMatch(pattern); - assertCursor(2, 2, "one\ntwo\nthree\n"); - assertTrue(match.isPresent()); - assertEquals("th", match.get().match()); - assertEquals(1, match.get().groupCount()); - assertEquals("h", match.get().group(1)); - - match = cursor.moveForwardToEndOfMatch(pattern); - assertCursor(2, 2, "one\ntwo\nthree\n"); - assertFalse(match.isPresent()); - - assertTrue(cursor.skipBackward("h")); - assertCursor(2, 1, "one\ntwo\nthree\n"); - assertFalse(cursor.skipBackward("x")); - - assertTrue(cursor.skipForward("hre")); - assertCursor(2, 4, "one\ntwo\nthree\n"); - assertFalse(cursor.skipForward("x")); - - try { - cursor.moveTo(mark); - fail(); - } catch (IllegalArgumentException e) { - // expected - } - - mark = cursor.createMark(); - cursor.moveToStartOfBuffer(); - assertEquals(new Position(0, 0), cursor.getPosition()); - cursor.moveTo(mark); - assertEquals(new Position(2, 4), cursor.getPosition()); - - cursor.moveTo(1, 2); - assertCursor(1, 2, "one\ntwo\nthree\n"); - - cursor.deleteSuffix(); - assertCursor(1, 2, "one\ntw\nthree\n"); - - cursor.deletePrefix(); - assertCursor(1, 0, "one\n\nthree\n"); - - cursor.deleteLine(); - assertCursor(1, 0, "one\nthree\n"); - - cursor.deleteLine(); - assertCursor(1, 0, "one\n"); - - cursor.deleteLine(); - assertCursor(1, 0, "one\n"); - - cursor.moveToStartOfBuffer().moveForward().writeNewlineAfter(); - assertCursor(0, 1, "o\nne\n"); - - cursor.deleteAll().writeLines("one", "two", "three", "four"); - cursor.moveToStartOfBuffer().moveToStartOfNextLine(); - assertCursor(1, 0, "one\ntwo\nthree\nfour\n"); - Pattern pattern2 = Pattern.compile("(o)(.)?"); - int count = cursor.replaceMatches(pattern2, m -> { - String prefix = m.group(2) == null ? "" : m.group(2); - return prefix + m.match() + m.group(1); - }); - assertCursor(3, 5, "one\ntwoo\nthree\nfuouor\n"); - assertEquals(2, count); - - cursor.moveToStartOfBuffer().moveToEndOfLine(); - Pattern pattern3 = Pattern.compile("o"); - count = cursor.replaceMatches(pattern3, m -> "a"); - assertEquals(4, count); - assertCursor(3, 5, "one\ntwaa\nthree\nfuauar\n"); - } - - private void assertCursor(int lineIndex, int columnIndex, String text) { - assertEquals(text, cursor.getBufferText()); - assertEquals(new Position(lineIndex, columnIndex), cursor.getPosition()); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java deleted file mode 100644 index 15fb36dc3d5..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/TextBufferImplTest.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.editor; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class TextBufferImplTest { - private final TextBufferImpl textBuffer = new TextBufferImpl(); - - @Test - void testWrite() { - assertEquals("", textBuffer.getString()); - assertWrite(2, 0, "foo\nbar\n", - 0, 0, "foo\nbar\n"); - - assertWrite(1, 6, "fofirst\nsecondo\nbar\n", - 0, 2, "first\nsecond"); - - assertWrite(3, 1, "fofirst\nsecondo\nbar\na", - 3, 0, "a"); - assertWrite(4, 0, "fofirst\nsecondo\nbar\na\n", - 3, 1, "\n"); - } - - @Test - void testDelete() { - write(0, 0, "foo\nbar\nzoo\n"); - delete(0, 2, 2, 1); - assertEquals("fooo\n", textBuffer.getString()); - - delete(0, 4, 1, 0); - assertEquals("fooo", textBuffer.getString()); - - delete(0, 0, 0, 4); - assertEquals("", textBuffer.getString()); - - delete(0, 0, 0, 0); - assertEquals("", textBuffer.getString()); - } - - private void assertWrite(int expectedLineIndex, int expectedColumnIndex, String expectedString, - int lineIndex, int columnIndex, String text) { - Position position = write(lineIndex, columnIndex, text); - assertEquals(new Position(expectedLineIndex, expectedColumnIndex), position); - assertEquals(expectedString, textBuffer.getString()); - } - - private Position write(int lineIndex, int columnIndex, String text) { - return textBuffer.write(new Position(lineIndex, columnIndex), text); - } - - private void delete(int startLineIndex, int startColumnIndex, - int endLineIndex, int endColumnIndex) { - textBuffer.delete(new Position(startLineIndex, startColumnIndex), - new Position(endLineIndex, endColumnIndex)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java deleted file mode 100644 index 507bf706484..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class DiskSizeTest { - - @Test - void bytes_to_display_count_test() { - assertEquals("-1 bytes", DiskSize.of(-1).asString()); - assertEquals("123 bytes", DiskSize.of(123).asString()); - assertEquals("1 kB", DiskSize.of(1_000).asString()); - assertEquals("15 MB", DiskSize.of(15_000_000).asString()); - assertEquals("123 GB", DiskSize.of(123_456_789_012L).asString()); - assertEquals("988 TB", DiskSize.of(987_654_321_098_765L).asString()); - assertEquals("987.7 TB", DiskSize.of(987_654_321_098_765L).asString(1)); - assertEquals("987.65 TB", DiskSize.of(987_654_321_098_765L).asString(2)); - assertEquals("2 PB", DiskSize.of(2_000_000_000_000_000L).asString()); - assertEquals("9 EB", DiskSize.of(Long.MAX_VALUE).asString()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java deleted file mode 100644 index 9a651494854..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import java.nio.file.FileSystem; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class EditorTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final UnixPath path = new UnixPath(fileSystem.getPath("/file")); - - @Test - void testEdit() { - path.writeUtf8File(joinLines("first", "second", "third")); - - LineEditor lineEditor = mock(LineEditor.class); - when(lineEditor.edit(any())).thenReturn( - LineEdit.none(), // don't edit the first line - LineEdit.remove(), // remove the second - LineEdit.replaceWith("replacement")); // replace the third - - Editor editor = new Editor(path.toPath(), lineEditor); - TaskContext context = mock(TaskContext.class); - - assertTrue(editor.converge(context)); - - verify(lineEditor, times(3)).edit(any()); - - // Verify the system modification message - ArgumentCaptor<String> modificationMessage = ArgumentCaptor.forClass(String.class); - verify(context).recordSystemModification(any(), modificationMessage.capture()); - assertEquals( - "Patching file /file:\n-second\n-third\n+replacement\n", - modificationMessage.getValue()); - - // Verify the new contents of the file: - assertEquals(joinLines("first", "replacement"), path.readUtf8File()); - } - - @Test - void testInsert() { - path.writeUtf8File(joinLines("second", "eight", "fifth", "seventh")); - - LineEditor lineEditor = mock(LineEditor.class); - when(lineEditor.edit(any())).thenReturn( - LineEdit.insertBefore("first"), // insert first, and keep the second line - LineEdit.replaceWith("third", "fourth"), // remove eight, and replace with third and fourth instead - LineEdit.none(), // Keep fifth - LineEdit.insert(List.of("sixth"), // insert sixth before seventh - List.of("eight"))); // add eight after seventh - - Editor editor = new Editor(path.toPath(), lineEditor); - TaskContext context = mock(TaskContext.class); - - assertTrue(editor.converge(context)); - - // Verify the system modification message - ArgumentCaptor<String> modificationMessage = ArgumentCaptor.forClass(String.class); - verify(context).recordSystemModification(any(), modificationMessage.capture()); - assertEquals( - "Patching file /file:\n" + - "+first\n" + - "-eight\n" + - "+third\n" + - "+fourth\n" + - "+sixth\n" + - "+eight\n", - modificationMessage.getValue()); - - // Verify the new contents of the file: - assertEquals(joinLines("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eight"), - path.readUtf8File()); - } - - @Test - void noop() { - path.writeUtf8File("line\n"); - - LineEditor lineEditor = mock(LineEditor.class); - when(lineEditor.edit(any())).thenReturn(LineEdit.none()); - - Editor editor = new Editor(path.toPath(), lineEditor); - TaskContext context = mock(TaskContext.class); - - assertFalse(editor.converge(context)); - - verify(lineEditor, times(1)).edit(any()); - - // Verify the system modification message - verify(context, times(0)).recordSystemModification(any(), any()); - - // Verify same contents - assertEquals("line\n", path.readUtf8File()); - } - - @Test - void testMissingFile() { - LineEditor lineEditor = mock(LineEditor.class); - when(lineEditor.onComplete()).thenReturn(List.of("line")); - - TaskContext context = mock(TaskContext.class); - var editor = new Editor(path.toPath(), lineEditor); - editor.converge(context); - - assertEquals("line\n", path.readUtf8File()); - } - - private static String joinLines(String... lines) { - return String.join("\n", lines) + "\n"; - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java deleted file mode 100644 index 8559e36fe8b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -public class FileAttributesCacheTest { - @Test - void exists() { - UnixPath unixPath = mock(UnixPath.class); - FileAttributesCache cache = new FileAttributesCache(unixPath); - - when(unixPath.getAttributesIfExists()).thenReturn(Optional.empty()); - assertFalse(cache.get().isPresent()); - verify(unixPath, times(1)).getAttributesIfExists(); - verifyNoMoreInteractions(unixPath); - - FileAttributes attributes = new FileAttributes(Instant.EPOCH, 0, 0, "", false, false, 0, 0, 0); - when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); - when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); - assertTrue(cache.get().isPresent()); - verify(unixPath, times(1 + 1)).getAttributesIfExists(); - verifyNoMoreInteractions(unixPath); - - assertEquals(attributes, cache.getOrThrow()); - verifyNoMoreInteractions(unixPath); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java deleted file mode 100644 index ed183738ef0..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.jupiter.api.Test; - -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMajor; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMinor; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -class FileAttributesTest { - - @Test - void parse_dev_t() { - assertEquals(0x12345BCD, deviceMajor(0x1234567890ABCDEFL)); - assertEquals(0x67890AEF, deviceMinor(0x1234567890ABCDEFL)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java deleted file mode 100644 index e1cea37ccbc..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileContentCacheTest.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.time.Instant; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -public class FileContentCacheTest { - private final UnixPath unixPath = mock(UnixPath.class); - private final FileContentCache cache = new FileContentCache(unixPath); - - private final byte[] content = "content".getBytes(StandardCharsets.UTF_8); - private final byte[] newContent = "new-content".getBytes(StandardCharsets.UTF_8); - - @Test - void get() { - when(unixPath.readBytes()).thenReturn(content); - assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1)).readBytes(); - verifyNoMoreInteractions(unixPath); - - // cache hit - assertArrayEquals(content, cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1)).readBytes(); - verifyNoMoreInteractions(unixPath); - - // cache miss - when(unixPath.readBytes()).thenReturn(newContent); - assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1))); - verify(unixPath, times(1 + 1)).readBytes(); - verifyNoMoreInteractions(unixPath); - - // cache hit both at times 0 and 1 - assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(0))); - verify(unixPath, times(1 + 1)).readBytes(); - verifyNoMoreInteractions(unixPath); - assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(1))); - verify(unixPath, times(1 + 1)).readBytes(); - verifyNoMoreInteractions(unixPath); - } - - @Test - void updateWith() { - cache.updateWith(content, Instant.ofEpochMilli(2)); - assertArrayEquals(content, cache.get(Instant.ofEpochMilli(2))); - verifyNoMoreInteractions(unixPath); - - cache.updateWith(newContent, Instant.ofEpochMilli(4)); - assertArrayEquals(newContent, cache.get(Instant.ofEpochMilli(4))); - verifyNoMoreInteractions(unixPath); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java deleted file mode 100644 index f7fb66fca94..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileDeleterTest.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -public class FileDeleterTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final UnixPath path = new UnixPath(fileSystem.getPath("/tmp/foo")); - private final FileDeleter deleter = new FileDeleter(path.toPath()); - private final TaskContext context = mock(TaskContext.class); - - @Test - void deleteExisting() { - assertFalse(deleter.converge(context)); - path.createParents().writeUtf8File("bar"); - assertTrue(deleter.converge(context)); - assertFalse(deleter.converge(context)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java deleted file mode 100644 index 76941d3333b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.time.Instant; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Set.of; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - -/** - * @author freva - */ - -public class FileFinderTest { - - @Nested - public class GeneralLogicTests { - - private final FileSystem fileSystem = TestFileSystem.create(); - - @Test - void all_files_non_recursive() { - assertFileHelper(FileFinder.files(testRoot()) - .maxDepth(1), - - of("file-1.json", "test.json", "test.txt"), - of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void all_files_recursive() { - assertFileHelper(FileFinder.files(testRoot()), - - of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json", "test/subdir-1/test"), - of("test", "test/subdir-1", "test/subdir-2")); - } - - @Test - void all_files_recursive_with_prune_relative() { - assertFileHelper(FileFinder.files(testRoot()).prune(fileSystem.getPath("test")), - - of("file-1.json", "test.json", "test.txt"), - of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void all_files_recursive_with_prune_absolute() { - assertFileHelper(FileFinder.files(testRoot()).prune(testRoot().resolve("test/subdir-1")), - - of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json"), - of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void throws_if_prune_path_not_under_base_path() { - assertThrows(IllegalArgumentException.class, () -> { - FileFinder.files(Path.of("/some/path")).prune(Path.of("/other/path")); - }); - } - - @Test - void with_file_filter_recursive() { - assertFileHelper(FileFinder.files(testRoot()) - .match(FileFinder.nameEndsWith(".json")), - - of("file-1.json", "test.json", "test/data.json"), - of("test.txt", "test", "test/file.txt", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void all_files_limited_depth() { - assertFileHelper(FileFinder.files(testRoot()) - .maxDepth(2), - - of("test.txt", "file-1.json", "test.json", "test/file.txt", "test/data.json"), - of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); - } - - @Test - void directory_with_filter() { - assertFileHelper(FileFinder.directories(testRoot()) - .match(FileFinder.nameStartsWith("subdir")) - .maxDepth(2), - - of("test/subdir-1", "test/subdir-2"), - of("file-1.json", "test.json", "test.txt", "test", "test/file.txt", "test/data.json")); - } - - @Test - void match_file_and_directory_with_same_name() { - assertFileHelper(FileFinder.from(testRoot()) - .match(FileFinder.nameEndsWith("test")), - - of("test", "test/subdir-1/test"), - of("file-1.json", "test.json", "test.txt")); - } - - @Test - void all_contents() { - assertFileHelper(FileFinder.from(testRoot()) - .maxDepth(1), - - of("file-1.json", "test.json", "test.txt", "test"), - of()); - - assertTrue(Files.exists(testRoot())); - } - - @BeforeEach - public void setup() throws IOException { - Path root = testRoot(); - Files.createDirectories(root); - - Files.createFile(root.resolve("file-1.json")); - Files.createFile(root.resolve("test.json")); - Files.createFile(root.resolve("test.txt")); - - Files.createDirectories(root.resolve("test")); - Files.createFile(root.resolve("test/file.txt")); - Files.createFile(root.resolve("test/data.json")); - - Files.createDirectories(root.resolve("test/subdir-1")); - Files.createFile(root.resolve("test/subdir-1/test")); - - Files.createDirectories(root.resolve("test/subdir-2")); - } - - private Path testRoot() { - return fileSystem.getPath("/file-finder"); - } - - private void assertFileHelper(FileFinder fileFinder, Set<String> expectedList, Set<String> expectedContentsAfterDelete) { - Set<String> actualList = fileFinder.stream() - .map(FileFinder.FileAttributes::path) - .map(testRoot()::relativize) - .map(Path::toString) - .collect(Collectors.toSet()); - assertEquals(expectedList, actualList); - - fileFinder.deleteRecursively(mock(TaskContext.class)); - Set<String> actualContentsAfterDelete = recursivelyListContents(testRoot()).stream() - .map(testRoot()::relativize) - .map(Path::toString) - .collect(Collectors.toSet()); - assertEquals(expectedContentsAfterDelete, actualContentsAfterDelete); - } - - private List<Path> recursivelyListContents(Path basePath) { - try (Stream<Path> pathStream = Files.list(basePath)) { - List<Path> paths = new LinkedList<>(); - pathStream.forEach(path -> { - paths.add(path); - if (Files.isDirectory(path)) - paths.addAll(recursivelyListContents(path)); - }); - return paths; - } catch (NoSuchFileException e) { - return List.of(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } - - @Nested - public class FilterUnitTests { - - private final BasicFileAttributes attributes = mock(BasicFileAttributes.class); - - @Test - void age_filter_test() { - Path path = Path.of("/my/fake/path"); - when(attributes.lastModifiedTime()).thenReturn(FileTime.from(Instant.now().minus(Duration.ofHours(1)))); - FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); - - assertFalse(FileFinder.olderThan(Duration.ofMinutes(61)).test(fileAttributes)); - assertTrue(FileFinder.olderThan(Duration.ofMinutes(59)).test(fileAttributes)); - - assertTrue(FileFinder.youngerThan(Duration.ofMinutes(61)).test(fileAttributes)); - assertFalse(FileFinder.youngerThan(Duration.ofMinutes(59)).test(fileAttributes)); - } - - @Test - void size_filters() { - Path path = Path.of("/my/fake/path"); - when(attributes.size()).thenReturn(100L); - FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); - - assertFalse(FileFinder.largerThan(101).test(fileAttributes)); - assertTrue(FileFinder.largerThan(99).test(fileAttributes)); - - assertTrue(FileFinder.smallerThan(101).test(fileAttributes)); - assertFalse(FileFinder.smallerThan(99).test(fileAttributes)); - } - - @Test - void filename_filters() { - Path path = Path.of("/my/fake/path/some-12352-file.json"); - FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); - - assertTrue(FileFinder.nameStartsWith("some-").test(fileAttributes)); - assertFalse(FileFinder.nameStartsWith("som-").test(fileAttributes)); - - assertTrue(FileFinder.nameEndsWith(".json").test(fileAttributes)); - assertFalse(FileFinder.nameEndsWith("file").test(fileAttributes)); - - assertTrue(FileFinder.nameMatches(Pattern.compile("some-[0-9]+-file.json")).test(fileAttributes)); - assertTrue(FileFinder.nameMatches(Pattern.compile("^some-[0-9]+-file.json$")).test(fileAttributes)); - assertFalse(FileFinder.nameMatches(Pattern.compile("some-[0-9]-file.json")).test(fileAttributes)); - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java deleted file mode 100644 index e418833ab50..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileMoverTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.FileSystem; -import java.nio.file.NoSuchFileException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.mock; - -/** - * @author hakonhall - */ -class FileMoverTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final TaskContext context = mock(TaskContext.class); - private final UnixPath source = new UnixPath(fileSystem.getPath("/from/source")); - private final UnixPath destination = new UnixPath(fileSystem.getPath("/to/destination")); - private final FileMover mover = new FileMover(source.toPath(), destination.toPath()); - - @Test - void movingRegularFile() { - assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/from/source"); - - source.createParents().writeUtf8File("content"); - assertConvergeThrows(() -> mover.converge(context), NoSuchFileException.class, "/to/destination"); - - destination.createParents(); - assertTrue(mover.converge(context)); - assertFalse(source.exists()); - assertTrue(destination.exists()); - assertEquals("content", destination.readUtf8File()); - - assertFalse(mover.converge(context)); - - source.writeUtf8File("content 2"); - assertConvergeThrows(() -> mover.converge(context), FileAlreadyExistsException.class, "/to/destination"); - - mover.replaceExisting(); - assertTrue(mover.converge(context)); - - source.writeUtf8File("content 3"); - destination.deleteIfExists(); - destination.createDirectory(); - assertTrue(mover.converge(context)); - } - - private void assertConvergeThrows(Runnable runnable, Class<?> expectedRootExceptionClass, String expectedMessage) { - try { - runnable.run(); - fail(); - } catch (Throwable t) { - Throwable rootCause = t; - do { - Throwable cause = rootCause.getCause(); - if (cause == null) break; - rootCause = cause; - } while (true); - - assertTrue(expectedRootExceptionClass.isInstance(rootCause), "Unexpected root cause: " + rootCause); - assertEquals(expectedMessage, rootCause.getMessage()); - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java deleted file mode 100644 index b0992e9826a..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author hakonhall - */ -public class FileSnapshotTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final UnixPath path = new UnixPath(fileSystem.getPath("/var/lib/file.txt")); - - private FileSnapshot fileSnapshot = FileSnapshot.forPath(path.toPath()); - - @Test - void fileDoesNotExist() { - assertFalse(fileSnapshot.exists()); - assertFalse(fileSnapshot.attributes().isPresent()); - assertFalse(fileSnapshot.content().isPresent()); - assertEquals(path.toPath(), fileSnapshot.path()); - } - - @Test - void directory() { - path.createParents().createDirectory(); - fileSnapshot = fileSnapshot.snapshot(); - assertTrue(fileSnapshot.exists()); - assertTrue(fileSnapshot.attributes().isPresent()); - assertTrue(fileSnapshot.attributes().get().isDirectory()); - } - - @Test - void regularFile() { - path.createParents().writeUtf8File("file content"); - fileSnapshot = fileSnapshot.snapshot(); - assertTrue(fileSnapshot.exists()); - assertTrue(fileSnapshot.attributes().isPresent()); - assertTrue(fileSnapshot.attributes().get().isRegularFile()); - assertTrue(fileSnapshot.utf8Content().isPresent()); - assertEquals("file content", fileSnapshot.utf8Content().get()); - - FileSnapshot newFileSnapshot = fileSnapshot.snapshot(); - assertSame(fileSnapshot, newFileSnapshot); - } - - @Test - void fileRemoval() { - path.createParents().writeUtf8File("file content"); - fileSnapshot = fileSnapshot.snapshot(); - assertTrue(fileSnapshot.exists()); - path.deleteIfExists(); - fileSnapshot = fileSnapshot.snapshot(); - assertFalse(fileSnapshot.exists()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java deleted file mode 100644 index c60de78bf8c..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -public class FileSyncTest { - private final TestTaskContext taskContext = new TestTaskContext(); - private final FileSystem fileSystem = TestFileSystem.create(); - - private final Path path = fileSystem.getPath("/dir/file.txt"); - private final UnixPath unixPath = new UnixPath(path); - private final FileSync fileSync = new FileSync(path); - - private String content = "content"; - private int ownerId = 123; // default is 1 - private int groupId = 456; // default is 2 - private String permissions = "rw-r-xr--"; - - @Test - void trivial() { - assertConvergence("Creating file /dir/file.txt with permissions rw-r-xr--", - "Changing user ID of /dir/file.txt from 1 to 123", - "Changing group ID of /dir/file.txt from 2 to 456"); - - content = "new-content"; - assertConvergence("Patching file /dir/file.txt"); - - ownerId = 124; - assertConvergence("Changing user ID of /dir/file.txt from 123 to 124"); - - groupId = 457; - assertConvergence("Changing group ID of /dir/file.txt from 456 to 457"); - - permissions = "rwxr--rwx"; - assertConvergence("Changing permissions of /dir/file.txt from rw-r-xr-- to " + - permissions); - } - - private void assertConvergence(String... systemModificationMessages) { - PartialFileData fileData = PartialFileData.builder() - .withContent(content) - .withOwnerId(ownerId) - .withGroupId(groupId) - .withPermissions(permissions) - .create(); - taskContext.clearSystemModificationLog(); - assertTrue(fileSync.convergeTo(taskContext, fileData)); - - assertTrue(Files.isRegularFile(path)); - fileData.getContent().ifPresent(content -> assertArrayEquals(content, unixPath.readBytes())); - fileData.getOwnerId().ifPresent(owner -> assertEquals((int) owner, unixPath.getOwnerId())); - fileData.getGroupId().ifPresent(group -> assertEquals((int) group, unixPath.getGroupId())); - fileData.getPermissions().ifPresent(permissions -> assertEquals(permissions, unixPath.getPermissions())); - - List<String> actualMods = taskContext.getSystemModificationLog(); - List<String> expectedMods = List.of(systemModificationMessages); - assertEquals(expectedMods, actualMods); - - UnixPath unixPath = new UnixPath(path); - Instant lastModifiedTime = unixPath.getLastModifiedTime(); - taskContext.clearSystemModificationLog(); - assertFalse(fileSync.convergeTo(taskContext, fileData)); - assertEquals(lastModifiedTime, unixPath.getLastModifiedTime()); - - actualMods = taskContext.getSystemModificationLog(); - assertEquals(new ArrayList<>(), actualMods); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java deleted file mode 100644 index 1264206bef3..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.test.file.TestFileSystem; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.time.Instant; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -public class FileWriterTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final TaskContext context = mock(TaskContext.class); - - @Test - void testWrite() { - final String content = "content"; - final String permissions = "rwxr-xr-x"; - final int owner = 123; - final int group = 456; - - Path path = fileSystem.getPath("/opt/vespa/tmp/file.txt"); - FileWriter writer = new FileWriter(path, () -> content) - .withPermissions(permissions) - .withOwnerId(owner) - .withGroupId(group) - .onlyIfFileDoesNotAlreadyExist(); - assertTrue(writer.converge(context)); - verify(context, times(1)).recordSystemModification(any(), eq("Creating file " + path + " with permissions rwxr-xr-x")); - - UnixPath unixPath = new UnixPath(path); - assertEquals(content, unixPath.readUtf8File()); - assertEquals(permissions, unixPath.getPermissions()); - assertEquals(owner, unixPath.getOwnerId()); - assertEquals(group, unixPath.getGroupId()); - Instant fileTime = unixPath.getLastModifiedTime(); - - // Second time is a no-op. - assertFalse(writer.converge(context)); - assertEquals(fileTime, unixPath.getLastModifiedTime()); - } - - @Test - void testAtomicWrite() { - FileWriter writer = new FileWriter(fileSystem.getPath("/foo/bar")) - .atomicWrite(true); - - assertTrue(writer.converge(context, "content")); - - verify(context).recordSystemModification(any(), eq("Creating file /foo/bar")); - assertEquals("content", new UnixPath(writer.path()).readUtf8File()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java deleted file mode 100644 index 11675bbe46f..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.UncheckedIOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author hakonhall - */ -public class MakeDirectoryTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final TestTaskContext context = new TestTaskContext(); - - private final String path = "/parent/dir"; - private String permissions = "rwxr----x"; - private int ownerId = 123; - private int groupId = 456; - - @Test - void newDirectory() { - verifySystemModifications( - "Creating directory " + path, - "Changing user ID of /parent/dir from 1 to 123", - "Changing group ID of /parent/dir from 2 to 456"); - - ownerId = 124; - verifySystemModifications("Changing user ID of /parent/dir from 123 to 124"); - - groupId = 457; - verifySystemModifications("Changing group ID of /parent/dir from 456 to 457"); - - permissions = "--x---r--"; - verifySystemModifications("Changing permissions of /parent/dir from rwxr----x to --x---r--"); - } - - private void verifySystemModifications(String... modifications) { - context.clearSystemModificationLog(); - MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)) - .createParents() - .withPermissions(permissions) - .withOwnerId(ownerId) - .withGroupId(groupId); - assertTrue(makeDirectory.converge(context)); - - assertEquals(List.of(modifications), context.getSystemModificationLog()); - - context.clearSystemModificationLog(); - assertFalse(makeDirectory.converge(context)); - assertEquals(List.of(), context.getSystemModificationLog()); - } - - @Test - void exceptionIfMissingParent() { - String path = "/parent/dir"; - MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)); - - try { - makeDirectory.converge(context); - } catch (UncheckedIOException e) { - if (e.getCause() instanceof NoSuchFileException) { - return; - } - throw e; - } - fail(); - } - - @Test - void okIfParentExists() { - String path = "/dir"; - MakeDirectory makeDirectory = new MakeDirectory(fileSystem.getPath(path)); - assertTrue(makeDirectory.converge(context)); - assertTrue(Files.isDirectory(fileSystem.getPath(path))); - - MakeDirectory makeDirectory2 = new MakeDirectory(fileSystem.getPath(path)); - assertFalse(makeDirectory2.converge(context)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java deleted file mode 100644 index 79fa1cf6ea2..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredBooleanTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * @author hakonhall - */ -public class StoredBooleanTest { - private final TaskContext context = mock(TaskContext.class); - private final FileSystem fileSystem = TestFileSystem.create(); - private final Path path = fileSystem.getPath("/foo"); - private final StoredBoolean storedBoolean = new StoredBoolean(path); - - @Test - void storedBoolean() { - assertFalse(storedBoolean.value()); - storedBoolean.set(context); - assertTrue(storedBoolean.value()); - storedBoolean.clear(context); - assertFalse(storedBoolean.value()); - } - - @Test - void testCompatibility() throws IOException { - StoredInteger storedInteger = new StoredInteger(path); - assertFalse(storedBoolean.value()); - - storedInteger.write(context, 1); - assertTrue(storedBoolean.value()); - - storedInteger.write(context, 2); - assertTrue(storedBoolean.value()); - - storedInteger.write(context, 0); - assertFalse(storedBoolean.value()); - - Files.delete(path); - assertFalse(storedBoolean.value()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java deleted file mode 100644 index d9dfcefc7e3..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.file.FileSystem; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -public class TemplateTest { - - @Test - void basic() { - FileSystem fileSystem = TestFileSystem.create(); - Path templatePath = fileSystem.getPath("/example.vm"); - String templateContent = "a $x, $y b"; - new UnixPath(templatePath).writeUtf8File(templateContent); - - Path toPath = fileSystem.getPath("/example"); - TaskContext taskContext = mock(TaskContext.class); - boolean converged = Template.at(templatePath) - .set("x", "foo") - .set("y", "bar") - .getFileWriterTo(toPath) - .converge(taskContext); - - assertTrue(converged); - - String actualContent = new UnixPath(toPath).readUtf8File(); - assertEquals("a foo, bar b", actualContent); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java deleted file mode 100644 index 5892a9b9f53..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author hakonhall - */ -public class UnixPathTest { - - private final FileSystem fs = TestFileSystem.create(); - - @Test - void createParents() { - Path parentDirectory = fs.getPath("/a/b/c"); - Path filePath = parentDirectory.resolve("bar"); - UnixPath path = new UnixPath(filePath); - - assertFalse(Files.exists(fs.getPath("/a"))); - path.createParents(); - assertTrue(Files.exists(parentDirectory)); - } - - @Test - void utf8File() { - String original = "foo\nbar\n"; - UnixPath path = new UnixPath(fs.getPath("example.txt")); - path.writeUtf8File(original); - String fromFile = path.readUtf8File(); - assertEquals(original, fromFile); - assertEquals(List.of("foo", "bar"), path.readLines()); - } - - @Test - void touch() { - UnixPath path = new UnixPath(fs.getPath("example.txt")); - assertTrue(path.create()); - assertEquals("", path.readUtf8File()); - assertFalse(path.create()); - } - - @Test - void permissions() { - String expectedPermissions = "rwxr-x---"; - UnixPath path = new UnixPath(fs.getPath("file.txt")); - path.writeUtf8File("foo"); - path.setPermissions(expectedPermissions); - assertEquals(expectedPermissions, path.getPermissions()); - } - - @Test - void badPermissionsString() { - assertThrows(IllegalArgumentException.class, () -> { - new UnixPath(fs.getPath("file.txt")).setPermissions("abcdefghi"); - }); - } - - @Test - void owner() { - Path path = fs.getPath("file.txt"); - UnixPath unixPath = new UnixPath(path); - unixPath.writeUtf8File("foo"); - - unixPath.setOwnerId(123); - assertEquals(123, unixPath.getOwnerId()); - - unixPath.setGroupId(456); - assertEquals(456, unixPath.getGroupId()); - } - - @Test - void createDirectoryWithPermissions() { - Path path = fs.getPath("dir"); - UnixPath unixPath = new UnixPath(path); - String permissions = "rwxr-xr--"; - assertTrue(unixPath.createDirectory(permissions)); - assertTrue(unixPath.isDirectory()); - assertEquals(permissions, unixPath.getPermissions()); - assertFalse(unixPath.createDirectory(permissions)); - } - - @Test - void createSymbolicLink() { - String original = "foo\nbar\n"; - UnixPath path = new UnixPath(fs.getPath("example.txt")); - path.writeUtf8File(original); - String fromFile = path.readUtf8File(); - assertEquals(original, fromFile); - - UnixPath link = path.createSymbolicLink(fs.getPath("link-to-example.txt")); - assertEquals(original, link.readUtf8File()); - } - - @Test - void readBytesIfExists() { - UnixPath path = new UnixPath(fs.getPath("example.txt")); - assertFalse(path.readBytesIfExists().isPresent()); - path.writeBytes(new byte[]{42}); - assertArrayEquals(new byte[]{42}, path.readBytesIfExists().get()); - } - - @Test - void deleteRecursively() throws Exception { - // Create the following file tree: - // - // /dir1 - // |--- dir2 - // |--- file1 - // /link1 -> /dir1/dir2 - // - var dir1 = fs.getPath("/dir1"); - var dir2 = dir1.resolve("dir2"); - var file1 = dir2.resolve("file1"); - Files.createDirectories(dir2); - Files.writeString(file1, "file1"); - var link1 = Files.createSymbolicLink(fs.getPath("/link1"), dir2); - - new UnixPath(link1).deleteRecursively(); - assertTrue(Files.exists(dir2), "Deleting " + link1 + " recursively does not remove " + dir2); - assertTrue(Files.exists(file1), "Deleting " + link1 + " recursively does not remove " + file1); - - new UnixPath(dir1).deleteRecursively(); - assertFalse(Files.exists(file1), dir1 + " deleted recursively"); - assertFalse(Files.exists(dir2), dir1 + " deleted recursively"); - assertFalse(Files.exists(dir1), dir1 + " deleted recursively"); - } - - @Test - void isEmptyDirectory() { - var path = new UnixPath((fs.getPath("/foo"))); - assertFalse(path.isEmptyDirectory()); - - path.writeUtf8File(""); - assertFalse(path.isEmptyDirectory()); - - path.deleteIfExists(); - path.createDirectory(); - assertTrue(path.isEmptyDirectory()); - - path.resolve("bar").writeUtf8File(""); - assertFalse(path.isEmptyDirectory()); - } - - @Test - void atomicWrite() { - var path = new UnixPath(fs.getPath("/dir/foo")); - path.createParents(); - path.writeUtf8File("bar"); - path.atomicWriteBytes("bar v2".getBytes(StandardCharsets.UTF_8)); - assertEquals("bar v2", path.readUtf8File()); - } - - @Test - void testParentAndFilename() { - var absolutePath = new UnixPath("/foo/bar"); - assertEquals("/foo", absolutePath.getParent().toString()); - assertEquals("bar", absolutePath.getFilename()); - - var pathWithoutSlash = new UnixPath("foo"); - assertRuntimeException(IllegalStateException.class, "Path has no parent directory: 'foo'", pathWithoutSlash::getParent); - assertEquals("foo", pathWithoutSlash.getFilename()); - - var pathWithSlash = new UnixPath("/foo"); - assertEquals("/", pathWithSlash.getParent().toString()); - assertEquals("foo", pathWithSlash.getFilename()); - - assertRuntimeException(IllegalStateException.class, "Path has no parent directory: '/'", () -> new UnixPath("/").getParent()); - assertRuntimeException(IllegalStateException.class, "Path has no filename: '/'", () -> new UnixPath("/").getFilename()); - } - - private <T extends RuntimeException> void assertRuntimeException(Class<T> baseClass, String message, Runnable runnable) { - try { - runnable.run(); - fail("No exception was thrown"); - } catch (RuntimeException e) { - if (!baseClass.isInstance(e)) { - fail("Exception class mismatch " + baseClass.getName() + " != " + e.getClass().getName()); - } - - assertEquals(message, e.getMessage()); - } - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java deleted file mode 100644 index 37fe90209ea..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author freva - */ -class ContainerFileSystemTest { - - private final FileSystem fileSystem = TestFileSystem.create(); - private final UnixPath containerRootOnHost = new UnixPath(fileSystem.getPath("/data/storage/ctr1")); - private final UserScope userScope = UserScope.create(new UserNamespace(10_000, 11_000, 10000)); - private final ContainerFileSystem containerFs = ContainerFileSystem.create(containerRootOnHost.createDirectories().toPath(), userScope); - - @Test - public void creates_files_and_directories_with_container_root_as_owner() throws IOException { - ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/opt/vespa/logs/file"), userScope.root()); - UnixPath unixPath = new UnixPath(containerPath).createParents().writeUtf8File("hello world"); - - for (ContainerPath p = containerPath; p.getParent() != null; p = p.getParent()) - assertOwnership(p, 0, 0, 10000, 11000); - - unixPath.setOwnerId(500).setGroupId(1000); - assertOwnership(containerPath, 500, 1000, 10500, 12000); - - UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); - ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/copy1"), userScope.root()); - Files.copy(hostFile.toPath(), destination); - assertOwnership(destination, 0, 0, 10000, 11000); - } - - @Test - public void file_write_and_read() throws IOException { - ContainerPath containerPath = ContainerPath.fromPathInContainer(containerFs, Path.of("/file"), userScope.root()); - UnixPath unixPath = new UnixPath(containerPath); - unixPath.writeUtf8File("hello"); - assertOwnership(containerPath, 0, 0, 10000, 11000); - - unixPath.setOwnerId(500).setGroupId(200); - assertOwnership(containerPath, 500, 200, 10500, 11200); - Files.writeString(containerPath, " world", StandardOpenOption.APPEND); - assertOwnership(containerPath, 500, 200, 10500, 11200); // Owner should not have been updated as the file already existed - - assertEquals("hello world", unixPath.readUtf8File()); - - unixPath.deleteIfExists(); - new UnixPath(containerPath.withUser(userScope.vespa())).writeUtf8File("test123"); - assertOwnership(containerPath, 1000, 1000, 11000, 12000); - } - - @Test - public void copy() throws IOException { - UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); - ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest"), userScope.root()); - - // If file is copied to JimFS path, the UID/GIDs are not fixed - Files.copy(hostFile.toPath(), destination.pathOnHost()); - assertEquals(String.valueOf(userScope.namespace().overflowId()), Files.getOwner(destination).getName()); - Files.delete(destination); - - Files.copy(hostFile.toPath(), destination); - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group on both source host file and destination container file - hostFile.setOwnerId(5).setGroupId(10); - new UnixPath(destination).setOwnerId(500).setGroupId(200); - assertOwnership(destination, 500, 200, 10500, 11200); - // Copy the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING - Files.copy(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - // The destination is recreated, so the owner should be root - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group and copy within ContainerFS - new UnixPath(destination).setOwnerId(500).setGroupId(200); - ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2"), userScope.root()); - Files.copy(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - assertOwnership(destination2, 500, 200, 10500, 11200); - } - - @Test - public void move() throws IOException { - UnixPath hostFile = new UnixPath(fileSystem.getPath("/file")).createNewFile(); - ContainerPath destination = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest"), userScope.root()); - - // If file is moved to JimFS path, the UID/GIDs are not fixed - Files.move(hostFile.toPath(), destination.pathOnHost()); - assertEquals(String.valueOf(userScope.namespace().overflowId()), Files.getOwner(destination).getName()); - Files.delete(destination); - - hostFile.createNewFile(); - Files.move(hostFile.toPath(), destination); - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group on both source host file and destination container file - hostFile.createNewFile(); - hostFile.setOwnerId(5).setGroupId(10); - new UnixPath(destination).setOwnerId(500).setGroupId(200); - assertOwnership(destination, 500, 200, 10500, 11200); - // Move the host file to destination again with COPY_ATTRIBUTES and REPLACE_EXISTING - Files.move(hostFile.toPath(), destination, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - // The destination is recreated, so the owner should be root - assertOwnership(destination, 0, 0, 10000, 11000); - - // Set owner + group and move within ContainerFS - new UnixPath(destination).setOwnerId(500).setGroupId(200); - ContainerPath destination2 = ContainerPath.fromPathInContainer(containerFs, Path.of("/dest2"), userScope.root()); - Files.move(destination, destination2, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - assertOwnership(destination2, 500, 200, 10500, 11200); - } - - @Test - public void symlink() throws IOException { - ContainerPath source = ContainerPath.fromPathInContainer(containerFs, Path.of("/src"), userScope.root()); - // Symlink from ContainerPath to some relative path (different FS provider) - Files.createSymbolicLink(source, fileSystem.getPath("../relative/target")); - assertEquals(fileSystem.getPath("../relative/target"), Files.readSymbolicLink(source)); - Files.delete(source); - - // Symlinks from ContainerPath to a ContainerPath: Target is resolved within container with base FS provider - Files.createSymbolicLink(source, ContainerPath.fromPathInContainer(containerFs, Path.of("/path/in/container"), userScope.root())); - assertEquals(fileSystem.getPath("/path/in/container"), Files.readSymbolicLink(source)); - assertOwnership(source, 0, 0, 10000, 11000); - } - - @Test - public void disallow_operations_on_symlink() throws IOException { - Path destination = fileSystem.getPath("/dir/file"); - Files.createDirectories(destination.getParent()); - - ContainerPath link = containerFs.getPath("/link"); - Files.createSymbolicLink(link, destination); - - // Cannot write file via symlink - assertThrows(IOException.class, () -> Files.writeString(link, "hello")); - - assertOwnership(link, 0, 0, 10_000, 11_000); - Files.setAttribute(link, "unix:uid", 10); // This succeeds because attribute is set on the link (destination does not exist) - assertFalse(Files.exists(destination)); - assertOwnership(link, 10, 0, 10_010, 11_000); - } - - @Test - public void disallow_operations_on_parent_symlink() throws IOException { - Path destination = fileSystem.getPath("/dir/sub/folder"); - Files.createDirectories(destination.getParent()); - - // Create symlink /some/dir/link -> /dir/sub - ContainerPath link = containerFs.getPath("/some/dir/link"); - Files.createDirectories(link.getParent()); - Files.createSymbolicLink(link, destination.getParent()); - - ContainerPath file = link.resolve("file"); - assertThrows(IOException.class, () -> Files.writeString(file, "hello")); - Files.writeString(file.pathOnHost(), "hello"); // Writing through host FS works - } - - @Test - public void permissions() throws IOException { - assertPermissions(Files.createDirectory(containerFs.getPath("/dir1")), "rwxr-x---"); - assertPermissions(Files.createDirectory(containerFs.getPath("/dir2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); - - assertPermissions(Files.createDirectories(containerFs.getPath("/sub/dir/leaf"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); - assertPermissions(containerFs.getPath("/sub/dir"), "r-x-w-rw-"); // Non-leafs get the same permission as the leaf - - // TODO: Uncomment when JimFS forwards attributes for SecureDirectoryStream::newByteChannel -// assertPermissions(Files.createFile(containerFs.getPath("/file1")), "rw-r-----"); -// assertPermissions(Files.createFile(containerFs.getPath("/file2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); - } - - private static void assertOwnership(ContainerPath path, int contUid, int contGid, int hostUid, int hostGid) throws IOException { - assertOwnership(path, contUid, contGid); - assertOwnership(path.pathOnHost(), hostUid, hostGid); - } - - private static void assertOwnership(Path path, int uid, int gid) throws IOException { - Map<String, Object> attrs = Files.readAttributes(path, "unix:*", LinkOption.NOFOLLOW_LINKS); - assertEquals(uid, attrs.get("uid")); - assertEquals(gid, attrs.get("gid")); - } - - private static void assertPermissions(Path path, String expected) throws IOException { - String actual = PosixFilePermissions.toString(Files.getPosixFilePermissions(path)); - assertEquals(expected, actual); - } - - private static FileAttribute<?> permissionsFromString(String permissions) { - return PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(permissions)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java deleted file mode 100644 index eb7a8e13925..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.function.Executable; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * @author freva - */ -class ContainerPathTest { - - private final FileSystem baseFs = TestFileSystem.create(); - private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), mock(UserScope.class)); - - @Test - public void create_new_container_path() { - ContainerPath path = fromPathInContainer(Path.of("/opt/vespa//logs/./file")); - assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file"); - - path = fromPathOnHost(baseFs.getPath("/data/storage/ctr1/opt/vespa/logs/file")); - assertPaths(path, "/data/storage/ctr1/opt/vespa/logs/file", "/opt/vespa/logs/file"); - - path = fromPathOnHost(baseFs.getPath("/data/storage/ctr2/..////./ctr1/./opt")); - assertPaths(path, "/data/storage/ctr1/opt", "/opt"); - - assertThrows(() -> fromPathInContainer(Path.of("relative/path")), "Path in container must be absolute: relative/path"); - assertThrows(() -> fromPathOnHost(baseFs.getPath("relative/path")), "Paths have different roots: /data/storage/ctr1, relative/path"); - assertThrows(() -> fromPathOnHost(baseFs.getPath("/data/storage/ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); - assertThrows(() -> fromPathOnHost(baseFs.getPath("/data/storage/ctr1/../ctr2")), "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); - } - - @Test - public void container_path_operations() { - ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs/file")); - ContainerPath parent = path.getParent(); - assertPaths(path.getRoot(), "/data/storage/ctr1", "/"); - assertPaths(parent, "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs"); - assertNull(path.getRoot().getParent()); - - assertEquals(Path.of("file"), path.getFileName()); - assertEquals(Path.of("logs"), path.getName(2)); - assertEquals(4, path.getNameCount()); - assertEquals(Path.of("vespa/logs"), path.subpath(1, 3)); - - assertTrue(path.startsWith(path)); - assertTrue(path.startsWith(parent)); - assertFalse(parent.startsWith(path)); - assertFalse(path.startsWith(Path.of(path.toString()))); - - assertTrue(path.endsWith(Path.of(path.pathInContainer()))); - assertTrue(path.endsWith(Path.of("logs/file"))); - assertFalse(path.endsWith(Path.of("/logs/file"))); - } - - @Test - public void resolution() { - ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs")); - assertPaths(path.resolve(Path.of("/root")), "/data/storage/ctr1/root", "/root"); - assertPaths(path.resolve(Path.of("relative")), "/data/storage/ctr1/opt/vespa/logs/relative", "/opt/vespa/logs/relative"); - assertPaths(path.resolve(Path.of("/../../../dir2/../../../dir2")), "/data/storage/ctr1/dir2", "/dir2"); - assertPaths(path.resolve(Path.of("/some/././///path")), "/data/storage/ctr1/some/path", "/some/path"); - - assertPaths(path.resolve(Path.of("../dir")), "/data/storage/ctr1/opt/vespa/dir", "/opt/vespa/dir"); - assertEquals(path.resolve(Path.of("../dir")), path.resolveSibling("dir")); - } - - @Test - public void resolves_real_paths() throws IOException { - ContainerPath path = fromPathInContainer(Path.of("/opt/vespa/logs")); - Files.createDirectories(path.pathOnHost().getParent()); - - Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/vespa/target1")); - Files.createSymbolicLink(path.pathOnHost(), path.pathOnHost().resolveSibling("target1")); - assertPaths(path.toRealPath(LinkOption.NOFOLLOW_LINKS), "/data/storage/ctr1/opt/vespa/logs", "/opt/vespa/logs"); - assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/vespa/target1", "/opt/vespa/target1"); - - Files.delete(path.pathOnHost()); - Files.createFile(baseFs.getPath("/data/storage/ctr1/opt/target2")); - Files.createSymbolicLink(path.pathOnHost(), baseFs.getPath("../target2")); - assertPaths(path.toRealPath(), "/data/storage/ctr1/opt/target2", "/opt/target2"); - - Files.delete(path.pathOnHost()); - Files.createFile(baseFs.getPath("/data/storage/ctr2")); - Files.createSymbolicLink(path.pathOnHost(), path.getRoot().pathOnHost().resolveSibling("ctr2")); - assertThrows(path::toRealPath, "Path /data/storage/ctr2 is not under container root /data/storage/ctr1"); - } - - private ContainerPath fromPathInContainer(Path pathInContainer) { - return ContainerPath.fromPathInContainer(containerFs, pathInContainer, UnixUser.ROOT); - } - private ContainerPath fromPathOnHost(Path pathOnHost) { - return ContainerPath.fromPathOnHost(containerFs, pathOnHost, UnixUser.ROOT); - } - - private static void assertPaths(ContainerPath actual, String expectedPathOnHost, String expectedPathInContainer) { - assertEquals(expectedPathOnHost, actual.pathOnHost().toString()); - assertEquals(expectedPathInContainer, actual.pathInContainer()); - } - - private static void assertThrows(Executable executable, String expectedMsg) { - String actualMsg = Assertions.assertThrows(IllegalArgumentException.class, executable).getMessage(); - assertEquals(expectedMsg, actualMsg); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java deleted file mode 100644 index 525c6d9162c..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerUserPrincipalLookupServiceTest.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.fs; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserNamespace; -import com.yahoo.vespa.hosted.node.admin.nodeagent.UserScope; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.attribute.UserPrincipalNotFoundException; - -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerGroupPrincipal; -import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerUserPrincipalLookupService.ContainerUserPrincipal; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author freva - */ -class ContainerUserPrincipalLookupServiceTest { - - private final UserScope userScope = UserScope.create(new UserNamespace(10_000, 11_000, 10000)); - private final ContainerUserPrincipalLookupService userPrincipalLookupService = - new ContainerUserPrincipalLookupService(TestFileSystem.create().getUserPrincipalLookupService(), userScope); - - @Test - public void correctly_resolves_ids() throws IOException { - ContainerUserPrincipal user = userPrincipalLookupService.lookupPrincipalByName("1000"); - assertEquals("vespa", user.getName()); - assertEquals("11000", user.baseFsPrincipal().getName()); - assertEquals(user, userPrincipalLookupService.lookupPrincipalByName("vespa")); - - ContainerGroupPrincipal group = userPrincipalLookupService.lookupPrincipalByGroupName("1000"); - assertEquals("vespa", group.getName()); - assertEquals("12000", group.baseFsPrincipal().getName()); - assertEquals(group, userPrincipalLookupService.lookupPrincipalByGroupName("vespa")); - - assertThrows(UserPrincipalNotFoundException.class, () -> userPrincipalLookupService.lookupPrincipalByName("test")); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java deleted file mode 100644 index 299d3e4b441..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import com.google.common.net.InetAddresses; - -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author smorgrav - */ -public class IPAddressesMock implements IPAddresses { - - private final Map<String, List<InetAddress>> otherAddresses = new HashMap<>(); - - public IPAddressesMock addAddress(String hostname, String ip) { - List<InetAddress> addresses = otherAddresses.getOrDefault(hostname, new ArrayList<>()); - addresses.add(InetAddresses.forString(ip)); - otherAddresses.put(hostname, addresses); - return this; - } - - @Override - public InetAddress[] getAddresses(String hostname) { - List<InetAddress> addresses = otherAddresses.get(hostname); - if (addresses == null) return new InetAddress[0]; - return addresses.toArray(new InetAddress[addresses.size()]); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java deleted file mode 100644 index 59ddc1f6c8d..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import com.google.common.net.InetAddresses; -import org.junit.jupiter.api.Test; - -import java.net.Inet6Address; -import java.net.InetAddress; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author smorgrav - */ -public class IPAddressesTest { - - private final IPAddressesMock mock = new IPAddressesMock(); - - @Test - void choose_sitelocal_ipv4_over_public() { - mock.addAddress("localhost", "38.3.4.2") - .addAddress("localhost", "10.0.2.2") - .addAddress("localhost", "fe80::1") - .addAddress("localhost", "2001::1"); - - assertEquals(InetAddresses.forString("10.0.2.2"), mock.getIPv4Address("localhost").get()); - } - - @Test - void choose_ipv6_public_over_local() { - mock.addAddress("localhost", "38.3.4.2") - .addAddress("localhost", "10.0.2.2") - .addAddress("localhost", "fe80::1") - .addAddress("localhost", "2001::1"); - - assertEquals(InetAddresses.forString("2001::1"), mock.getIPv6Address("localhost").get()); - } - - @Test - void throws_when_multiple_ipv6_addresses() { - assertThrows(RuntimeException.class, () -> { - mock.addAddress("localhost", "2001::1") - .addAddress("localhost", "2001::2"); - mock.getIPv6Address("localhost"); - }); - } - - @Test - void throws_when_multiple_private_ipv4_addresses() { - assertThrows(RuntimeException.class, () -> { - mock.addAddress("localhost", "38.3.4.2") - .addAddress("localhost", "10.0.2.2") - .addAddress("localhost", "10.0.2.3"); - mock.getIPv4Address("localhost"); - }); - } - - @Test - void translator_with_valid_parameters() { - - // Test simplest possible address - Inet6Address original = (Inet6Address) InetAddresses.forString("2001:db8::1"); - Inet6Address prefix = (Inet6Address) InetAddresses.forString("fd00::"); - InetAddress translated = IPAddresses.prefixTranslate(original, prefix, 8); - assertEquals("fd00:0:0:0:0:0:0:1", translated.getHostAddress()); - - - // Test an actual aws address we use - original = (Inet6Address) InetAddresses.forString("2600:1f16:f34:5300:ccc6:1703:b7c2:369d"); - translated = IPAddresses.prefixTranslate(original, prefix, 8); - assertEquals("fd00:0:0:0:ccc6:1703:b7c2:369d", translated.getHostAddress()); - - // Test different subnet size - translated = IPAddresses.prefixTranslate(original, prefix, 6); - assertEquals("fd00:0:0:5300:ccc6:1703:b7c2:369d", translated.getHostAddress()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java deleted file mode 100644 index 69d5c6f2c31..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.network; - -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; - -/** - * @author gjoranv - */ -public class VersionedIpAddressTest { - - @Test - void ip4_address_can_be_generated_from_string() { - var ip4 = VersionedIpAddress.from("10.0.0.1"); - assertEquals(IPVersion.IPv4, ip4.version()); - assertEquals("10.0.0.1", ip4.asString()); - } - - @Test - void ip6_address_can_be_generated_from_string() { - var ip6 = VersionedIpAddress.from("::1"); - assertEquals(IPVersion.IPv6, ip6.version()); - assertEquals("::1", ip6.asString()); - } - - @Test - void they_are_sorted_by_version_then_by_address() { - var ip4 = VersionedIpAddress.from("10.0.0.1"); - var ip4_2 = VersionedIpAddress.from("127.0.0.1"); - var ip6 = VersionedIpAddress.from("::1"); - var ip6_2 = VersionedIpAddress.from("::2"); - - var sorted = Stream.of(ip4_2, ip6, ip4, ip6_2) - .sorted() - .toList(); - assertEquals(List.of(ip6, ip6_2, ip4, ip4_2), sorted); - } - - @Test - void endpoint_with_port_is_generated_correctly_for_both_versions() { - var ip4 = VersionedIpAddress.from("10.0.0.1"); - var ip6 = VersionedIpAddress.from("::1"); - - assertEquals("10.0.0.1:8080", ip4.asEndpoint(8080)); - assertEquals("[::1]:8080", ip6.asEndpoint(8080)); - } - - @Test - void equals_and_hashCode_are_implemented() { - var one = VersionedIpAddress.from("::1"); - var two = VersionedIpAddress.from("::2"); - var local = VersionedIpAddress.from("127.0.0.1"); - var ten = VersionedIpAddress.from("10.0.0.1"); - assertEquals(one, VersionedIpAddress.from("::1")); - assertNotEquals(one, two); - assertNotEquals(one, local); - assertNotEquals(one, ten); - - assertEquals(local, VersionedIpAddress.from("127.0.0.1")); - assertNotEquals(local, two); - assertNotEquals(local, 10); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java deleted file mode 100644 index 19bc2d59bb2..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author hakonhall - */ -public class ChildProcess2ImplTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final Timer timer = mock(Timer.class); - private final CommandLine commandLine = mock(CommandLine.class); - private final ProcessApi2 processApi = mock(ProcessApi2.class); - private Path temporaryFile; - - @BeforeEach - public void setUp() throws IOException { - temporaryFile = Files.createTempFile(fileSystem.getPath("/"), "", ""); - } - - @Test - void testSuccess() throws Exception { - when(commandLine.getTimeout()).thenReturn(Duration.ofHours(1)); - when(commandLine.getMaxOutputBytes()).thenReturn(10L); - when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); - when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); - when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); - when(commandLine.toString()).thenReturn("program arg"); - - when(timer.currentTime()).thenReturn( - Instant.ofEpochMilli(1), - Instant.ofEpochMilli(2)); - - when(processApi.waitFor(anyLong(), any())).thenReturn(true); - - try (ChildProcess2Impl child = - new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { - child.waitForTermination(); - } - } - - @Test - void testTimeout() throws Exception { - when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); - when(commandLine.getMaxOutputBytes()).thenReturn(10L); - when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); - when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); - when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); - when(commandLine.toString()).thenReturn("program arg"); - - when(timer.currentTime()).thenReturn( - Instant.ofEpochSecond(0), - Instant.ofEpochSecond(2)); - - when(processApi.waitFor(anyLong(), any())).thenReturn(true); - - try (ChildProcess2Impl child = - new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { - try { - child.waitForTermination(); - fail(); - } catch (TimeoutChildProcessException e) { - assertEquals( - "Command 'program arg' timed out after PT1S: stdout/stderr: ''", - e.getMessage()); - } - } - } - - @Test - void testMaxOutputBytes() throws Exception { - when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); - when(commandLine.getMaxOutputBytes()).thenReturn(10L); - when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); - when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); - when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); - when(commandLine.toString()).thenReturn("program arg"); - - when(timer.currentTime()).thenReturn( - Instant.ofEpochMilli(0), - Instant.ofEpochMilli(1)); - - when(processApi.waitFor(anyLong(), any())).thenReturn(true); - - Files.writeString(temporaryFile, "1234567890123"); - - try (ChildProcess2Impl child = - new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { - try { - child.waitForTermination(); - fail(); - } catch (LargeOutputChildProcessException e) { - assertEquals( - "Command 'program arg' output more than 13 bytes: stdout/stderr: '1234567890123'", - e.getMessage()); - } - } - } - - @Test - void testUnkillable() throws Exception { - when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); - when(commandLine.getMaxOutputBytes()).thenReturn(10L); - when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); - when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); - when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); - when(commandLine.toString()).thenReturn("program arg"); - - when(timer.currentTime()).thenReturn( - Instant.ofEpochMilli(0), - Instant.ofEpochMilli(1)); - - when(processApi.waitFor(anyLong(), any())).thenReturn(false); - - Files.writeString(temporaryFile, "1234567890123"); - - try (ChildProcess2Impl child = - new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { - try { - child.waitForTermination(); - fail(); - } catch (UnkillableChildProcessException e) { - assertEquals( - "Command 'program arg' did not terminate even after SIGTERM, +PT2M, SIGKILL, and +PT3M: stdout/stderr: '1234567890123'", - e.getMessage()); - } - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java deleted file mode 100644 index fead96404a5..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Optional; -import java.util.function.Predicate; - -import static org.junit.jupiter.api.Assertions.*; - -public class CommandLineTest { - private final TestTerminal terminal = new TestTerminal(); - private final TestTaskContext context = new TestTaskContext(); - private final CommandLine commandLine = terminal.newCommandLine(context); - - @AfterEach - public void tearDown() { - terminal.verifyAllCommandsExecuted(); - } - - @Test - void testStrings() { - terminal.expectCommand( - "/bin/bash \"with space\" \"speci&l\" \"\" \"double\\\"quote\" 2>&1", - 0, - ""); - commandLine.add("/bin/bash", "with space", "speci&l", "", "double\"quote").execute(); - assertEquals("bash", commandLine.programName()); - } - - @Test - void testBasicExecute() { - terminal.expectCommand("foo bar 2>&1", 0, "line1\nline2\n\n"); - CommandResult result = commandLine.add("foo", "bar").execute(); - assertEquals(0, result.getExitCode()); - assertEquals("line1\nline2", result.getOutput()); - assertEquals("line1\nline2\n\n", result.getUntrimmedOutput()); - assertEquals(List.of("line1", "line2"), result.getOutputLines()); - assertEquals(1, context.getSystemModificationLog().size()); - assertEquals("Executing command: foo bar 2>&1", context.getSystemModificationLog().get(0)); - - List<CommandLine> commandLines = terminal.getTestProcessFactory().getMutableCommandLines(); - assertEquals(1, commandLines.size()); - assertEquals(commandLine, commandLines.get(0)); - - int lines = result.map(r -> r.getOutputLines().size()); - assertEquals(2, lines); - } - - @Test - void verifyDefaults() { - assertEquals(CommandLine.DEFAULT_TIMEOUT, commandLine.getTimeout()); - assertEquals(CommandLine.DEFAULT_MAX_OUTPUT_BYTES, commandLine.getMaxOutputBytes()); - assertEquals(CommandLine.DEFAULT_SIGTERM_GRACE_PERIOD, commandLine.getSigTermGracePeriod()); - assertEquals(CommandLine.DEFAULT_SIGKILL_GRACE_PERIOD, commandLine.getSigKillGracePeriod()); - assertEquals(0, commandLine.getArguments().size()); - assertEquals(Optional.empty(), commandLine.getOutputFile()); - assertEquals(StandardCharsets.UTF_8, commandLine.getOutputEncoding()); - assertTrue(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()); - Predicate<Integer> defaultExitCodePredicate = commandLine.getSuccessfulExitCodePredicate(); - assertTrue(defaultExitCodePredicate.test(0)); - assertFalse(defaultExitCodePredicate.test(1)); - } - - @Test - void executeSilently() { - terminal.ignoreCommand(""); - commandLine.add("foo", "bar").executeSilently(); - assertEquals(0, context.getSystemModificationLog().size()); - commandLine.recordSilentExecutionAsSystemModification(); - assertEquals(1, context.getSystemModificationLog().size()); - assertEquals("Executed command: foo bar 2>&1", context.getSystemModificationLog().get(0)); - } - - @Test - void processFactorySpawnFails() { - assertThrows(NegativeArraySizeException.class, () -> { - terminal.interceptCommand( - commandLine.toString(), - command -> { - throw new NegativeArraySizeException(); - }); - commandLine.add("foo").execute(); - }); - } - - @Test - void waitingForTerminationExceptionStillClosesChild() { - TestChildProcess2 child = new TestChildProcess2(0, ""); - child.throwInWaitForTermination(new NegativeArraySizeException()); - terminal.interceptCommand(commandLine.toString(), command -> child); - assertFalse(child.closeCalled()); - try { - commandLine.add("foo").execute(); - fail(); - } catch (NegativeArraySizeException e) { - // OK - } - - assertTrue(child.closeCalled()); - } - - @Test - void programFails() { - terminal.expectCommand("foo 2>&1", 1, ""); - try { - commandLine.add("foo").execute(); - fail(); - } catch (ChildProcessFailureException e) { - assertEquals( - "Command 'foo 2>&1' terminated with exit code 1: stdout/stderr: ''", - e.getMessage()); - } - } - - @Test - void mapException() { - terminal.ignoreCommand("output"); - CommandResult result = terminal.newCommandLine(context).add("program").execute(); - IllegalArgumentException exception = new IllegalArgumentException("foo"); - try { - result.mapOutput(output -> { - throw exception; - }); - fail(); - } catch (UnexpectedOutputException e) { - assertEquals("Command 'program 2>&1' output was not of the expected format: " + - "Failed to map output: stdout/stderr: 'output'", e.getMessage()); - assertEquals(e.getCause(), exception); - } - } - - @Test - void testMapEachLine() { - assertEquals( - 1 + 2 + 3, - terminal.ignoreCommand("1\n2\n3\n") - .newCommandLine(context) - .add("foo") - .execute() - .mapEachLine(Integer::valueOf) - .stream() - .mapToInt(i -> i) - .sum()); - } - - @Test - void addTokensWithMultipleWhiteSpaces() { - terminal.expectCommand("iptables -L 2>&1"); - commandLine.addTokens("iptables -L").execute(); - - terminal.verifyAllCommandsExecuted(); - } - - @Test - void addTokensWithSpecialCharacters() { - terminal.expectCommand("find . ! -name hei 2>&1"); - commandLine.addTokens("find . ! -name hei").execute(); - - terminal.verifyAllCommandsExecuted(); - } - - @Test - void testEnvironment() { - terminal.expectCommand("env k1=v1 -u k2 \"key 3=value 3\" programname 2>&1"); - commandLine.add("programname") - .setEnvironmentVariable("key 3", "value 3") - .removeEnvironmentVariable("k2") - .setEnvironmentVariable("k1", "v1") - .execute(); - terminal.verifyAllCommandsExecuted(); - } - - @Test - public void testToString() { - commandLine.add("bash", "-c", "echo", "$MY_SECRET"); - assertEquals("bash -c echo \"$MY_SECRET\" 2>&1", commandLine.toString()); - commandLine.censorArgument(); - assertEquals("bash -c echo <censored> 2>&1", commandLine.toString()); - - terminal.expectCommand("bash -c echo \"$MY_SECRET\" 2>&1"); - commandLine.execute(); - terminal.verifyAllCommandsExecuted(); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java deleted file mode 100644 index 58429f9f084..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.process; - -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.jdisc.test.TestTimer; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static com.yahoo.yolean.Exceptions.uncheck; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class ProcessFactoryImplTest { - private final ProcessStarter starter = mock(ProcessStarter.class); - private final TestTimer timer = new TestTimer(); - private final ProcessFactoryImpl processFactory = new ProcessFactoryImpl(starter, timer); - - @Test - void testSpawn() { - CommandLine commandLine = mock(CommandLine.class); - when(commandLine.getArguments()).thenReturn(List.of("program")); - when(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()).thenReturn(true); - when(commandLine.programName()).thenReturn("program"); - Path outputPath; - try (ChildProcess2Impl child = processFactory.spawn(commandLine)) { - outputPath = child.getOutputPath(); - assertTrue(Files.exists(outputPath)); - assertEquals("rw-------", new UnixPath(outputPath).getPermissions()); - ArgumentCaptor<ProcessBuilder> processBuilderCaptor = - ArgumentCaptor.forClass(ProcessBuilder.class); - verify(starter).start(processBuilderCaptor.capture()); - ProcessBuilder processBuilder = processBuilderCaptor.getValue(); - assertTrue(processBuilder.redirectErrorStream()); - ProcessBuilder.Redirect redirect = processBuilder.redirectOutput(); - assertEquals(ProcessBuilder.Redirect.Type.WRITE, redirect.type()); - assertEquals(outputPath.toFile(), redirect.file()); - } - - assertFalse(Files.exists(outputPath)); - } - - @Test - void testSpawnWithPersistentOutputFile() { - - class TemporaryFile implements AutoCloseable { - private final Path path; - - private TemporaryFile() { - String outputFileName = ProcessFactoryImplTest.class.getSimpleName() + "-temporary-test-file.out"; - FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute( - PosixFilePermissions.fromString("rw-------")); - path = uncheck(() -> Files.createTempFile(outputFileName, ".out", fileAttribute)); - } - - @Override - public void close() { - uncheck(() -> Files.deleteIfExists(path)); - } - } - - try (TemporaryFile outputPath = new TemporaryFile()) { - CommandLine commandLine = mock(CommandLine.class); - when(commandLine.getArguments()).thenReturn(List.of("program")); - when(commandLine.programName()).thenReturn("program"); - when(commandLine.getOutputFile()).thenReturn(Optional.of(outputPath.path)); - try (ChildProcess2Impl child = processFactory.spawn(commandLine)) { - assertEquals(outputPath.path, child.getOutputPath()); - assertTrue(Files.exists(outputPath.path)); - assertEquals("rw-------", new UnixPath(outputPath.path).getPermissions()); - } - - assertTrue(Files.exists(outputPath.path)); - } - - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java deleted file mode 100644 index f6a695ea003..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.systemd; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.mock; - -/** - * @author hakonhall - */ -public class SystemCtlTest { - - private final TaskContext taskContext = mock(TaskContext.class); - private final TestTerminal terminal = new TestTerminal(); - - @Test - void enable() { - terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "") - .expectCommand("systemctl enable docker 2>&1") - .expectCommand("systemctl --quiet is-enabled docker 2>&1"); - - SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); - assertTrue(enableDockerService.converge(taskContext)); - assertFalse(enableDockerService.converge(taskContext), "Already converged"); - } - - @Test - void enableCommandFailure() { - terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, "") - .expectCommand("systemctl enable docker 2>&1", 1, "error enabling service"); - SystemCtl.SystemCtlEnable enableDockerService = new SystemCtl(terminal).enable("docker"); - try { - enableDockerService.converge(taskContext); - fail(); - } catch (ChildProcessFailureException e) { - // success - } - } - - - @Test - void start() { - terminal.expectCommand( - "systemctl show docker 2>&1", - 0, - "a=b\n" + - "ActiveState=failed\n" + - "bar=zoo\n") - .expectCommand("systemctl start docker 2>&1", 0, ""); - - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); - assertTrue(startDockerService.converge(taskContext)); - } - - @Test - void startIsNoop() { - terminal.expectCommand( - "systemctl show docker 2>&1", - 0, - "a=b\n" + - "ActiveState=active\n" + - "bar=zoo\n") - .expectCommand("systemctl start docker 2>&1", 0, ""); - - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); - assertFalse(startDockerService.converge(taskContext)); - } - - - @Test - void startCommandFailre() { - terminal.expectCommand("systemctl show docker 2>&1", 1, "error"); - SystemCtl.SystemCtlStart startDockerService = new SystemCtl(terminal).start("docker"); - try { - startDockerService.converge(taskContext); - fail(); - } catch (ChildProcessFailureException e) { - // success - } - } - - - @Test - void disable() { - terminal.expectCommand("systemctl --quiet is-enabled docker 2>&1") - .expectCommand("systemctl disable docker 2>&1") - .expectCommand("systemctl --quiet is-enabled docker 2>&1", 1, ""); - - assertTrue(new SystemCtl(terminal).disable("docker").converge(taskContext)); - assertFalse(new SystemCtl(terminal).disable("docker").converge(taskContext), "Already converged"); - } - - @Test - void stop() { - terminal.expectCommand( - "systemctl show docker 2>&1", - 0, - "a=b\n" + - "ActiveState=active\n" + - "bar=zoo\n") - .expectCommand("systemctl stop docker 2>&1", 0, ""); - - assertTrue(new SystemCtl(terminal).stop("docker").converge(taskContext)); - } - - @Test - void restart() { - terminal.expectCommand("systemctl restart docker 2>&1", 0, ""); - assertTrue(new SystemCtl(terminal).restart("docker").converge(taskContext)); - } - - @Test - void testUnitExists() { - SystemCtl systemCtl = new SystemCtl(terminal); - - terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, - "UNIT FILE STATE\n" + - "\n" + - "0 unit files listed.\n"); - assertFalse(systemCtl.serviceExists(taskContext, "foo")); - - terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, - "UNIT FILE STATE \n" + - "foo.service enabled\n" + - "\n" + - "1 unit files listed.\n"); - assertTrue(systemCtl.serviceExists(taskContext, "foo")); - - terminal.expectCommand("systemctl list-unit-files foo.service 2>&1", 0, "garbage"); - try { - systemCtl.serviceExists(taskContext, "foo"); - fail(); - } catch (Exception e) { - assertTrue(e.getMessage().contains("garbage")); - } - } - - @Test - void withSudo() { - SystemCtl systemCtl = new SystemCtl(terminal).withSudo(); - terminal.expectCommand("sudo systemctl restart docker 2>&1", 0, ""); - assertTrue(systemCtl.restart("docker").converge(taskContext)); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java deleted file mode 100644 index 3fc10a38a99..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.systemd; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.function.Function; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class SystemCtlTesterTest { - - private static final String unit = "my-unit"; - private final TestTerminal terminal = new TestTerminal(); - private final SystemCtlTester systemCtl = new SystemCtlTester(terminal); - private final TestTaskContext context = new TestTaskContext(); - - @Test - void return_expectations() { - assertSystemCtlMethod(sct -> sct.expectEnable(unit), sc -> sc.enable(unit).converge(context)); - assertSystemCtlMethod(sct -> sct.expectDisable(unit), sc -> sc.disable(unit).converge(context)); - assertSystemCtlMethod(sct -> sct.expectStart(unit), sc -> sc.start(unit).converge(context)); - assertSystemCtlMethod(sct -> sct.expectStop(unit), sc -> sc.stop(unit).converge(context)); - assertSystemCtlMethod(sct -> sct.expectServiceExists(unit), sc -> sc.serviceExists(context, unit)); - assertSystemCtlMethod(sct -> sct.expectIsActive(unit), sc -> sc.isActive(context, unit)); - } - - @Test - void void_tests() { - systemCtl.expectRestart(unit); - systemCtl.restart(unit).converge(context); - terminal.verifyAllCommandsExecuted(); - - systemCtl.expectDaemonReload(); - systemCtl.daemonReload(context); - terminal.verifyAllCommandsExecuted(); - } - - private void assertSystemCtlMethod(Function<SystemCtlTester, SystemCtlTester.Expectation> systemCtlTesterExpectationFunction, - Function<SystemCtl, Boolean> systemCtlFunction) { - List.of(true, false).forEach(wantedReturnValue -> { - systemCtlTesterExpectationFunction.apply(systemCtl).andReturn(wantedReturnValue); - assertEquals(wantedReturnValue, systemCtlFunction.apply(systemCtl)); - terminal.verifyAllCommandsExecuted(); - }); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java deleted file mode 100644 index 1e2f69d7bc8..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.template; - -import org.junit.jupiter.api.Test; - -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author hakonhall - */ -public class TemplateTest { - @Test - void verifyNewlineRemoval() { - Template template = Template.from("a%{list a}\n" + - "b%{end}\n" + - "c%{list c-}\n" + - "d%{end-}\n" + - "e\n", - new TemplateDescriptor().setRemoveNewline(false)); - template.add("a"); - template.add("c"); - - assertEquals("a\n" + - "b\n" + - "cde\n", - template.render()); - } - - @Test - void verifyIfSection() { - Template template = Template.from("Hello%{if cond} world%{end}!"); - assertEquals("Hello world!", template.snapshot().set("cond", true).render()); - assertEquals("Hello!", template.snapshot().set("cond", false).render()); - } - - @Test - void verifyComplexIfSection() { - Template template = Template.from("%{if cond}\n" + - "var: %{=varname}\n" + - "if: %{if !inner}inner is false%{end-}\n" + - "list: %{list formname}element%{end-}\n" + - "%{end}\n"); - - assertEquals("", template.snapshot().set("cond", false).render()); - - assertEquals("var: varvalue\n" + - "if: \n" + - "list: \n", - template.snapshot() - .set("cond", true) - .set("varname", "varvalue") - .set("inner", true) - .render()); - - Template template2 = template.snapshot() - .set("cond", true) - .set("varname", "varvalue") - .set("inner", false); - template2.add("formname"); - - assertEquals("var: varvalue\n" + - "if: inner is false\n" + - "list: element\n", template2.render()); - } - - @Test - void verifyElse() { - var template = Template.from("%{if cond}\n" + - "if body\n" + - "%{else}\n" + - "else body\n" + - "%{end}\n"); - assertEquals("if body\n", template.snapshot().set("cond", true).render()); - assertEquals("else body\n", template.snapshot().set("cond", false).render()); - } - - @Test - void verifySnapshotPreservesList() { - var template = Template.from("%{list foo}hello %{=area}%{end}"); - template.add("foo") - .set("area", "world"); - - assertEquals("hello world", template.render()); - assertEquals("hello world", template.snapshot().render()); - - Template snapshot = template.snapshot(); - snapshot.add("foo") - .set("area", "Norway"); - assertEquals("hello worldhello Norway", snapshot.render()); - } - - @Test - void verifyVariableSection() { - Template template = getTemplate("template1.tmp"); - template.set("varname", "varvalue"); - assertEquals("variable section 'varvalue'\n" + - "end of text\n", template.render()); - } - - @Test - void verifySimpleListSection() { - Template template = getTemplate("template1.tmp"); - template.set("varname", "varvalue") - .add("listname") - .set("varname", "different varvalue") - .set("varname2", "varvalue2"); - assertEquals("variable section 'varvalue'\n" + - "same variable section 'different varvalue'\n" + - "different variable section 'varvalue2'\n" + - "between ends\n" + - "end of text\n", template.render()); - } - - @Test - void verifyNestedListSection() { - Template template = getTemplate("template2.tmp"); - ListElement A0 = template.add("listA"); - ListElement A0B0 = A0.add("listB"); - ListElement A0B1 = A0.add("listB"); - - ListElement A1 = template.add("listA"); - ListElement A1B0 = A1.add("listB"); - assertEquals("body A\n" + - "body B\n" + - "body B\n" + - "body A\n" + - "body B\n", - template.render()); - } - - @Test - void verifyVariableReferences() { - Template template = getTemplate("template3.tmp"); - template.set("varname", "varvalue") - .set("innerVarSetAtTop", "val2"); - template.add("l"); - template.add("l") - .set("varname", "varvalue2"); - assertEquals("varvalue\n" + - "varvalue\n" + - "inner varvalue\n" + - "val2\n" + - "inner varvalue2\n" + - "val2\n", - template.render()); - } - - @Test - void badTemplates() { - assertException(BadTemplateException.class, "Unknown section 'zoo' at line 2 and column 6", - () -> Template.from("foo\nbar%{zoo}")); - - assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", - () -> Template.from("%{=")); - - assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", - () -> Template.from("%{=¬atoken}")); - - assertException(BadTemplateException.class, "Expected identifier at line 1 and column 8", - () -> Template.from("%{list ¬atoken}")); - - assertException(BadTemplateException.class, "Missing end directive for section started at line 1 and column 12", - () -> Template.from("%{list foo}missing end")); - - assertException(BadTemplateException.class, "Stray 'end' at line 1 and column 3", - () -> Template.from("%{end}stray end")); - - assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 4 has not been set: notset", - () -> Template.from("%{=notset}").render()); - - assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 6 has not been set: cond", - () -> Template.from("%{if cond}%{end}").render()); - - assertException(NotBooleanValueTemplateException.class, "cond was set to a non-boolean value: must be true or false", - () -> Template.from("%{if cond}%{end}").set("cond", 1).render()); - - assertException(NoSuchNameTemplateException.class, "No such element 'listname' in the template section starting at " + - "line 1 and column 1, and ending at line 1 and column 4", - () -> Template.from("foo").add("listname")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the list section at line 1 and column 16 is in conflict with the identically " + - "named list section at line 1 and column 1", - () -> Template.from("%{list a}%{end}%{list a}%{end}")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the list section at line 1 and column 6 is in conflict with the identically " + - "named variable section at line 1 and column 1", - () -> Template.from("%{=a}%{list a}%{end}")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the variable section at line 1 and column 16 is in conflict with the identically " + - "named list section at line 1 and column 1", - () -> Template.from("%{list a}%{end}%{=a}")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the list section at line 1 and column 14 is in conflict with the identically " + - "named if section at line 1 and column 1", - () -> Template.from("%{if a}%{end}%{list a}%{end}")); - - assertException(NameAlreadyExistsTemplateException.class, - "The name 'a' of the if section at line 1 and column 16 is in conflict with the identically " + - "named list section at line 1 and column 1", - () -> Template.from("%{list a}%{end}%{if a}%{end}")); - } - - private <T extends Throwable> void assertException(Class<T> class_, String message, Runnable runnable) { - T exception = assertThrows(class_, runnable::run); - assertEquals(message, exception.getMessage()); - } - - private Template getTemplate(String filename) { - return Template.at(Path.of("src/test/resources/" + filename)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java deleted file mode 100644 index 505cf807116..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import org.junit.jupiter.api.Test; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author hakonhall - */ -public class YumPackageNameTest { - - @Test - void testBuilder() { - YumPackageName yumPackage = new YumPackageName.Builder("docker") - .setEpoch("2") - .setVersion("1.12.6") - .setRelease("71.git3e8e77d.el7.centos.1") - .setArchitecture("x86_64") - .build(); - assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName()); - } - - @Test - void testAllValidFormats() { - // name - verifyPackageName( - "docker-engine-selinux", - null, - "docker-engine-selinux", - null, - null, - null, - "docker-engine-selinux", - null); - - // name with parenthesis - verifyPackageName( - "dnf-command(versionlock)", - null, - "dnf-command(versionlock)", - null, - null, - null, - "dnf-command(versionlock)", - null); - - // name.arch - verifyPackageName( - "docker-engine-selinux.x86_64", - null, - "docker-engine-selinux", - null, - null, - "x86_64", - "docker-engine-selinux.x86_64", - null); - - // name-ver - verifyPackageName("docker-engine-selinux-1.12.6", - null, - "docker-engine-selinux", - "1.12.6", - null, - null, - "docker-engine-selinux-0:1.12.6", - null); - - // name-ver-rel - verifyPackageName("docker-engine-selinux-1.12.6-1.el7", - null, - "docker-engine-selinux", - "1.12.6", - "1.el7", - null, - "docker-engine-selinux-0:1.12.6-1.el7", - "docker-engine-selinux-0:1.12.6-1.el7.*"); - - // name-ver-rel.arch - verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64", - null, - "docker-engine-selinux", - "1.12.6", - "1.el7", - "x86_64", - "docker-engine-selinux-0:1.12.6-1.el7.x86_64", - "docker-engine-selinux-0:1.12.6-1.el7.*"); - - // name-epoch:ver-rel.arch - verifyPackageName( - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "2", - "docker", - "1.12.6", - "71.git3e8e77d.el7.centos.1", - "x86_64", - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); - - // epoch:name-ver-rel.arch - verifyPackageName( - "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "2", - "docker", - "1.12.6", - "71.git3e8e77d.el7.centos.1", - "x86_64", - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); - } - - private void verifyPackageName(String input, - String epoch, - String name, - String version, - String release, - String architecture, - String toName, - String toVersionName) { - YumPackageName yumPackageName = YumPackageName.fromString(input); - assertPackageField("epoch", epoch, yumPackageName.getEpoch()); - assertPackageField("name", name, Optional.of(yumPackageName.getName())); - assertPackageField("version", version, yumPackageName.getVersion()); - assertPackageField("release", release, yumPackageName.getRelease()); - assertPackageField("architecture", architecture, yumPackageName.getArchitecture()); - assertPackageField("toName()", toName, Optional.of(yumPackageName.toName())); - - if (toVersionName == null) { - try { - yumPackageName.toVersionLockName(); - fail(); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("Version is missing ") || - e.getMessage().contains("Release is missing "), - "Exception message contains expected substring: " + e.getMessage()); - } - } else { - assertEquals(toVersionName, yumPackageName.toVersionLockName()); - } - } - - private void assertPackageField(String field, String expected, Optional<String> actual) { - if (expected == null) { - assertFalse(actual.isPresent(), field + " is not present"); - } else { - assertEquals(expected, actual.get(), field + " has expected value"); - } - } - - @Test - void testArchitectures() { - assertEquals("x86_64", YumPackageName.fromString("docker.x86_64").getArchitecture().get()); - assertEquals("i686", YumPackageName.fromString("docker.i686").getArchitecture().get()); - assertEquals("noarch", YumPackageName.fromString("docker.noarch").getArchitecture().get()); - } - - @Test - void unrecognizedArchitectureGetsGobbledUp() { - YumPackageName packageName = YumPackageName.fromString("docker-engine-selinux-1.12.6-1.el7.i486"); - // This is not a great feature - please use YumPackageName.Builder instead. - assertEquals("1.el7.i486", packageName.getRelease().get()); - } - - @Test - void failParsingOfPackageNameWithEpochAndArchitecture() { - try { - YumPackageName.fromString("epoch:docker-engine-selinux-1.12.6-1.el7.x86_64"); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().toLowerCase().contains("epoch")); - } - } - - @Test - void testSubset() { - YumPackageName yumPackage = new YumPackageName.Builder("docker") - .setVersion("1.12.6") - .build(); - - assertTrue(yumPackage.isSubsetOf(yumPackage)); - assertTrue(yumPackage.isSubsetOf(new YumPackageName.Builder("docker") - .setVersion("1.12.6") - .setEpoch("2") - .setRelease("71.git3e8e77d.el7.centos.1") - .setArchitecture("x86_64") - .build())); - assertFalse(yumPackage.isSubsetOf(new YumPackageName.Builder("docker") - .setVersion("1.13.1") - .build())); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java deleted file mode 100644 index 27b23d26b24..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureException; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.mock; - -/** - * @author hakonhall - */ -public class YumTest { - - private final TaskContext taskContext = mock(TaskContext.class); - private final TestTerminal terminal = new TestTerminal(); - private final Yum yum = new Yum(terminal); - - @AfterEach - public void after() { - terminal.verifyAllCommandsExecuted(); - } - - @Test - void testQueryInstalled() { - terminal.expectCommand( - "rpm -q docker --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", - 0, - "docker\n2\n1.13.1\n74.git6e3bb8e.el7.centos\nx86_64"); - - Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "docker"); - - assertTrue(installed.isPresent()); - assertEquals("docker", installed.get().getName()); - assertEquals("2", installed.get().getEpoch().get()); - assertEquals("1.13.1", installed.get().getVersion().get()); - assertEquals("74.git6e3bb8e.el7.centos", installed.get().getRelease().get()); - assertEquals("x86_64", installed.get().getArchitecture().get()); - } - - @Test - void testQueryInstalledPartial() { - terminal.expectCommand( - "rpm -q vespa-node-admin --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", - 0, - "vespa-node-admin\n(none)\n6.283.62\n1.el7\nnoarch"); - - Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "vespa-node-admin"); - - assertTrue(installed.isPresent()); - assertEquals("vespa-node-admin", installed.get().getName()); - assertEquals("0", installed.get().getEpoch().get()); - assertEquals("6.283.62", installed.get().getVersion().get()); - assertEquals("1.el7", installed.get().getRelease().get()); - assertEquals("noarch", installed.get().getArchitecture().get()); - } - - @Test - void testQueryNotInstalled() { - terminal.expectCommand( - "rpm -q fake-package --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", - 1, - "package fake-package is not installed"); - - Optional<YumPackageName> installed = yum.queryInstalled(taskContext, "fake-package"); - - assertFalse(installed.isPresent()); - } - - @Test - void testQueryInstalledMultiplePackages() { - terminal.expectCommand( - "rpm -q kernel-devel --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", - 0, - "kernel-devel\n" + - "(none)\n" + - "4.18.0\n" + - "305.7.1.el8_4\n" + - "x86_64\n" + - "kernel-devel\n" + - "(none)\n" + - "4.18.0\n" + - "240.15.1.el8_3\n" + - "x86_64\n"); - try { - yum.queryInstalled(taskContext, "kernel-devel"); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - assertEquals("Found multiple installed packages for 'kernel-devel'. Version is required to match package exactly", e.getMessage()); - } - } - - @Test - void testAlreadyInstalled() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "foobar\nNothing to do.\n"); // Note trailing dot - assertFalse(yum.install("package-1", "package-2") - .enableRepo("repo1", "repo2") - .converge(taskContext)); - } - - @Test - void testAlreadyUpgraded() { - terminal.expectCommand( - "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1", - 0, - "foobar\nNothing to do.\n"); // Same message as yum install no-op - - assertFalse(yum.upgrade("package-1", "package-2") - .converge(taskContext)); - } - - @Test - void testAlreadyRemoved() { - mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1")); - terminal.expectCommand( - "yum remove --assumeyes package-1 package-2 2>&1", - 0, - "foobar\nNo packages marked for removal.\n"); // Different output - - assertFalse(yum.remove("package-1", "package-2") - .converge(taskContext)); - } - - @Test - void skipsYumRemoveNotInRpm() { - mockRpmQuery("package-1", null); - mockRpmQuery("package-2", null); - assertFalse(yum.remove("package-1", "package-2").converge(taskContext)); - } - - @Test - void testInstall() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "installing, installing"); - - assertTrue(yum - .install("package-1", "package-2") - .converge(taskContext)); - } - - @Test - void skipsYumInstallIfInRpm() { - mockRpmQuery("package-1-0:1.2.3-1", YumPackageName.fromString("package-1-1.2.3-1")); - mockRpmQuery("package-2", YumPackageName.fromString("1:package-2-1.2.3-1.el7.x86_64")); - assertFalse(yum.install("package-1-1.2.3-1", "package-2").converge(taskContext)); - } - - @Test - void testInstallWithEnablerepo() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "installing, installing"); - - assertTrue(yum - .install("package-1", "package-2") - .enableRepo("repo-name") - .converge(taskContext)); - } - - @Test - void testInstallWithEnablerepoDisablerepo() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes \"--disablerepo=*\" --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "installing, installing"); - - assertTrue(yum - .install("package-1", "package-2") - .enableRepo("repo-name") - .disableRepo("*") - .converge(taskContext)); - } - - @Test - void testWithVersionLock() { - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Last metadata expiration check: 0:51:26 ago on Thu 14 Jan 2021 09:39:24 AM UTC.\n"); - terminal.expectCommand("yum versionlock add --assumeyes \"openssh-0:8.0p1-4.el8_1.*\" 2>&1"); - terminal.expectCommand( - "yum install --assumeyes openssh-0:8.0p1-4.el8_1.x86_64 2>&1", - 0, - "installing"); - - YumPackageName pkg = new YumPackageName - .Builder("openssh") - .setVersion("8.0p1") - .setRelease("4.el8_1") - .setArchitecture("x86_64") - .build(); - assertTrue(yum.installFixedVersion(pkg).converge(taskContext)); - } - - @Test - void testWithDifferentVersionLock() { - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Repository chef_rpms-release is listed more than once in the configuration\n" + - "chef-0:12.21.1-1.el7.*\n" + - "package-0:0.1-8.el7.*\n"); - - terminal.expectCommand("yum versionlock delete \"package-0:0.1-8.el7.*\" 2>&1"); - - terminal.expectCommand("yum versionlock add --assumeyes --enablerepo=somerepo \"package-0:0.10-654.el7.*\" 2>&1"); - - terminal.expectCommand( - "yum install --assumeyes --enablerepo=somerepo package-0:0.10-654.el7 2>&1", - 0, - "Nothing to do\n"); - - - assertTrue(yum - .installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")) - .enableRepo("somerepo") - .converge(taskContext)); - } - - @Test - void testWithExistingVersionLock() { - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Repository chef_rpms-release is listed more than once in the configuration\n" + - "chef-0:12.21.1-1.el7.*\n" + - "package-0:0.10-654.el7.*\n"); - terminal.expectCommand( - "yum install --assumeyes package-0:0.10-654.el7 2>&1", - 0, - "Nothing to do\n"); - - assertFalse(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext)); - } - - @Test - void testWithDowngrade() { - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Repository chef_rpms-release is listed more than once in the configuration\n" + - "chef-0:12.21.1-1.el7.*\n" + - "package-0:0.10-654.el7.*\n"); - - terminal.expectCommand( - "yum install --assumeyes package-0:0.10-654.el7 2>&1", - 0, - "Package matching package-=.0.10-654.el7 already installed. Checking for update.\n" + - "Nothing to do\n"); - - terminal.expectCommand("yum downgrade --assumeyes package-0:0.10-654.el7 2>&1"); - - assertTrue(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7")).converge(taskContext)); - } - - @Test - void testFailedInstall() { - assertThrows(ChildProcessFailureException.class, () -> { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 1, - "error"); - - yum - .install("package-1", "package-2") - .enableRepo("repo-name") - .converge(taskContext); - fail(); - }); - } - - @Test - void testUnknownPackages() { - mockRpmQuery("package-1", null); - terminal.expectCommand( - "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 package-3 2>&1", - 0, - "Loaded plugins: fastestmirror, langpacks\n" + - "Loading mirror speeds from cached hostfile\n" + - "No package package-1 available.\n" + - "No package package-2 available.\n" + - "Nothing to do\n"); - - var command = yum.install("package-1", "package-2", "package-3"); - try { - command.converge(taskContext); - fail(); - } catch (Exception e) { - assertNotNull(e.getCause()); - assertEquals("Unknown package: package-1", e.getCause().getMessage()); - } - } - - @Test - void throwIfNoPackagesSpecified() { - assertThrows(IllegalArgumentException.class, () -> { - yum.install(); - }); - } - - @Test - void allowToCallUpgradeWithNoPackages() { - terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK"); - yum.upgrade().converge(taskContext); - } - - @Test - void testDeleteVersionLock() { - terminal.expectCommand("yum versionlock delete openssh-0:8.0p1-4.el8_1.x86_64 2>&1"); - - YumPackageName pkg = new YumPackageName - .Builder("openssh") - .setVersion("8.0p1") - .setRelease("4.el8_1") - .setArchitecture("x86_64") - .build(); - assertTrue(yum.deleteVersionLock(pkg).converge(taskContext)); - } - - private void mockRpmQuery(String packageName, YumPackageName installedOrNull) { - new YumTester(terminal).expectQueryInstalled(packageName).andReturn(installedOrNull); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java deleted file mode 100644 index aafa0fcfd72..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Optional; -import java.util.function.Function; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author freva - */ -public class YumTesterTest { - - private static final String[] packages = {"pkg1", "pkg2"}; - private static final String[] repos = {"repo1", "repo2"}; - private static final String[] disablerepos = {"disablerepo1", "disablerepo2"}; - private static final YumPackageName minimalPackage = YumPackageName.fromString("pkg-1.13.1-0.el7"); - private static final YumPackageName fullPackage = YumPackageName.fromString("2:pkg-1.13.1-0.el7.x86_64"); - - private final TestTerminal terminal = new TestTerminal(); - private final YumTester yum = new YumTester(terminal); - private final TestTaskContext context = new TestTaskContext(); - - @Test - void generic_yum_methods() { - assertYumMethod(yum -> yum.expectInstall(packages).withDisableRepo(disablerepos).withEnableRepo(repos), - yum -> yum.install(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); - - assertYumMethod(yum -> yum.expectUpdate(packages).withDisableRepo(disablerepos).withEnableRepo(repos), - yum -> yum.upgrade(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); - - assertYumMethod(yum -> yum.expectRemove(packages).withDisableRepo(disablerepos).withEnableRepo(repos), - yum -> yum.remove(List.of(packages)).disableRepo(disablerepos).enableRepo(repos).converge(context)); - - assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName()).withDisableRepo(disablerepos).withEnableRepo(repos), - yum -> yum.installFixedVersion(minimalPackage).disableRepo(disablerepos).enableRepo(repos).converge(context)); - - // versionlock always returns success - assertYumMethodAlwaysSuccess(yum -> yum.expectDeleteVersionLock(minimalPackage.toName()), - yum -> yum.deleteVersionLock(minimalPackage).converge(context)); - - } - - @Test - void disable_other_repos() { - assertYumMethod(yum -> yum.expectInstall(packages).withDisableRepo("*").withEnableRepo(repos), - yum -> yum.install(List.of(packages)).disableRepo("*").enableRepo(repos).converge(context)); - } - - @Test - void expect_query_installed() { - yum.expectQueryInstalled(packages[0]).andReturn(fullPackage); - assertEquals(Optional.of(fullPackage), yum.queryInstalled(context, packages[0])); - terminal.verifyAllCommandsExecuted(); - } - - private void assertYumMethod(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction, - Function<Yum, Boolean> yumFunction) { - List.of(true, false).forEach(wantedReturnValue -> { - yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue); - assertEquals(wantedReturnValue, yumFunction.apply(yum)); - terminal.verifyAllCommandsExecuted(); - }); - } - - private void assertYumMethodAlwaysSuccess(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction, - Function<Yum, Boolean> yumFunction) { - List.of(true, false).forEach(wantedReturnValue -> { - yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue); - assertEquals(true, yumFunction.apply(yum)); - terminal.verifyAllCommandsExecuted(); - }); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java deleted file mode 100644 index 7ac47aad1fa..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.config.provision.WireguardKeyWithTimestamp; -import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author gjoranv - */ -public class WireguardPeerTest { - - @Test - void peers_are_sorted_by_hostname_ascending() { - List<WireguardPeer> peers = Stream.of( - peer("b"), - peer("a"), - peer("c") - ).sorted().toList(); - - assertEquals("a", peers.get(0).hostname().value()); - assertEquals("b", peers.get(1).hostname().value()); - assertEquals("c", peers.get(2).hostname().value()); - } - - private static WireguardPeer peer(String hostname) { - return new WireguardPeer(HostName.of(hostname), List.of(VersionedIpAddress.from("::1:1")), - new WireguardKeyWithTimestamp(WireguardKey.generateRandomForTesting(), Instant.EPOCH)); - } - -} diff --git a/node-admin/src/test/resources/default-env-example.txt b/node-admin/src/test/resources/default-env-example.txt deleted file mode 100644 index debae073271..00000000000 --- a/node-admin/src/test/resources/default-env-example.txt +++ /dev/null @@ -1,5 +0,0 @@ -override VESPA_HOSTNAME myhostname -fallback VESPA_CONFIGSERVER fallback-configserver -fallback VESPA_TLS_CONFIG_FILE /fallback/path/to/config.file -unset VESPA_LEGACY_OPTION -fallback VESPA_LEGACY_OPTION duplicated-variable
\ No newline at end of file diff --git a/node-admin/src/test/resources/default-env-rewritten.txt b/node-admin/src/test/resources/default-env-rewritten.txt deleted file mode 100644 index 94a91f4e793..00000000000 --- a/node-admin/src/test/resources/default-env-rewritten.txt +++ /dev/null @@ -1,4 +0,0 @@ -override VESPA_HOSTNAME my-new-hostname -fallback VESPA_CONFIGSERVER new-fallback-configserver -override VESPA_TLS_CONFIG_FILE /override/path/to/config.file -unset VESPA_LEGACY_OPTION diff --git a/node-admin/src/test/resources/template1.tmp b/node-admin/src/test/resources/template1.tmp deleted file mode 100644 index d53c875a0f3..00000000000 --- a/node-admin/src/test/resources/template1.tmp +++ /dev/null @@ -1,10 +0,0 @@ -variable section '%{=varname}' -%{list listname} -same variable section '%{=varname}' -different variable section '%{=varname2}' -%{list innerlistname} -inner form text -%{end} -between ends -%{end} -end of text diff --git a/node-admin/src/test/resources/template2.tmp b/node-admin/src/test/resources/template2.tmp deleted file mode 100644 index d36cb4a4a48..00000000000 --- a/node-admin/src/test/resources/template2.tmp +++ /dev/null @@ -1,4 +0,0 @@ -%{list listA}body A -%{list listB}body B -%{end} -%{end} diff --git a/node-admin/src/test/resources/template3.tmp b/node-admin/src/test/resources/template3.tmp deleted file mode 100644 index 27566e72a9d..00000000000 --- a/node-admin/src/test/resources/template3.tmp +++ /dev/null @@ -1,6 +0,0 @@ -%{=varname} -%{=varname} -%{list l} -inner %{=varname} -%{=innerVarSetAtTop} -%{end} @@ -101,7 +101,6 @@ <module>model-evaluation</module> <module>model-integration</module> <module>node-repository</module> - <module>node-admin</module> <module>opennlp-linguistics</module> <module>orchestrator-restapi</module> <module>orchestrator</module> |