diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2021-10-14 16:20:12 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-10-14 16:20:12 +0200 |
commit | 3e52c48f560623c73bbdc3a02099657eea0cf958 (patch) | |
tree | 6055137406434de8fdc55885a37574b8cf57a2ef | |
parent | 88cd5ae7d2a2660ca8420a91824fa14d78e95582 (diff) | |
parent | 799868af5183de996487aaf2e49e0869c6e95c94 (diff) |
Merge pull request #19564 from vespa-engine/freva/prepare-container-fs
Prepare to use ContainerPaths
27 files changed, 166 insertions, 175 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Cgroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Cgroup.java index c7616abd508..1d87415b78e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Cgroup.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Cgroup.java @@ -5,12 +5,7 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.OptionalInt; import java.util.logging.Logger; @@ -85,17 +80,9 @@ public class Cgroup { } private OptionalInt readCgroupsCpuInt(UnixPath unixPath) { - final byte[] currentContentBytes; - try { - currentContentBytes = Files.readAllBytes(unixPath.toPath()); - } catch (NoSuchFileException e) { - return OptionalInt.empty(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - String currentContent = new String(currentContentBytes, StandardCharsets.UTF_8).strip(); - return OptionalInt.of(Integer.parseInt(currentContent)); + return unixPath.readUtf8FileIfExists() + .map(s -> OptionalInt.of(Integer.parseInt(s.strip()))) + .orElseGet(OptionalInt::empty); } private boolean writeCgroupsCpuInt(NodeAgentContext context, UnixPath unixPath, int value) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index ec043036c81..2c0fa439000 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -57,7 +56,7 @@ public class CoreCollector { return GDB_PATH_RHEL8; } - Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) { + String readBinPathFallback(NodeAgentContext context, Path coredumpPath) { String command = getGdbPath(context) + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; String[] wrappedCommand = {"/bin/sh", "-c", command}; CommandResult result = docker.executeCommandInContainerAsRoot(context, wrappedCommand); @@ -67,10 +66,10 @@ public class CoreCollector { throw new ConvergenceException(String.format("Failed to extract binary path from GDB, result: %s, command: %s", asString(result), Arrays.toString(wrappedCommand))); } - return Paths.get(matcher.group("path").split(" ")[0]); + return matcher.group("path").split(" ")[0]; } - Path readBinPath(NodeAgentContext context, Path coredumpPath) { + String readBinPath(NodeAgentContext context, Path coredumpPath) { String[] command = {"file", coredumpPath.toString()}; try { CommandResult result = docker.executeCommandInContainerAsRoot(context, command); @@ -80,12 +79,12 @@ public class CoreCollector { Matcher execfnMatcher = EXECFN_PATH_PATTERN.matcher(result.getOutput()); if (execfnMatcher.find()) { - return Paths.get(execfnMatcher.group("path").split(" ")[0]); + return execfnMatcher.group("path").split(" ")[0]; } Matcher fromMatcher = FROM_PATH_PATTERN.matcher(result.getOutput()); if (fromMatcher.find()) { - return Paths.get(fromMatcher.group("path").split(" ")[0]); + return fromMatcher.group("path").split(" ")[0]; } } catch (RuntimeException e) { context.log(logger, Level.WARNING, String.format("Failed getting bin path, command: %s. " + @@ -95,9 +94,9 @@ public class CoreCollector { return readBinPathFallback(context, coredumpPath); } - List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) { + List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, String binPath, boolean allThreads) { String threads = allThreads ? "thread apply all bt" : "bt"; - String[] command = {getGdbPath(context), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; + String[] command = {getGdbPath(context), "-n", "-ex", threads, "-batch", binPath, coredumpPath.toString()}; CommandResult result = docker.executeCommandInContainerAsRoot(context, command); if (result.getExitCode() != 0) @@ -106,8 +105,8 @@ public class CoreCollector { return List.of(result.getOutput().split("\n")); } - List<String> readJstack(NodeAgentContext context, Path coredumpPath, Path binPath) { - String[] command = {"jhsdb", "jstack", "--exe", binPath.toString(), "--core", coredumpPath.toString()}; + List<String> readJstack(NodeAgentContext context, Path coredumpPath, String binPath) { + String[] command = {"jhsdb", "jstack", "--exe", binPath, "--core", coredumpPath.toString()}; CommandResult result = docker.executeCommandInContainerAsRoot(context, command); if (result.getExitCode() != 0) @@ -128,10 +127,10 @@ public class CoreCollector { Map<String, Object> data = new HashMap<>(); try { - Path binPath = readBinPath(context, coredumpPath); + String binPath = readBinPath(context, coredumpPath); - data.put("bin_path", binPath.toString()); - if (binPath.getFileName().toString().equals("java")) { + data.put("bin_path", binPath); + if (Path.of(binPath).getFileName().toString().equals("java")) { data.put("backtrace_all_threads", readJstack(context, coredumpPath, binPath)); } else { data.put("backtrace", readBacktrace(context, coredumpPath, binPath, false)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index f812f4096bd..4c0ffa3dce9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -52,7 +52,7 @@ public class CoredumpHandler { private final Terminal terminal; private final CoreCollector coreCollector; private final CoredumpReporter coredumpReporter; - private final Path crashPatchInContainer; + private final String crashPatchInContainer; private final Path doneCoredumpsPath; private final int operatorGroupId; private final Metrics metrics; @@ -65,13 +65,13 @@ public class CoredumpHandler { * @param operatorGroupId group ID of the group that will be set as the owner of the processed coredump */ public CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter, - Path crashPathInContainer, Path doneCoredumpsPath, int operatorGroupId, Metrics metrics) { + String crashPathInContainer, Path doneCoredumpsPath, int operatorGroupId, Metrics metrics) { this(terminal, coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath, operatorGroupId, metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString()); } CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter, - Path crashPathInContainer, Path doneCoredumpsPath, int operatorGroupId, Metrics metrics, + String crashPathInContainer, Path doneCoredumpsPath, int operatorGroupId, Metrics metrics, Clock clock, Supplier<String> coredumpIdSupplier) { this.terminal = terminal; this.coreCollector = coreCollector; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index bd01b320666..280ebd86d13 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -62,14 +62,13 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private static final Duration REFRESH_PERIOD = Duration.ofDays(1); private static final Duration REFRESH_BACKOFF = Duration.ofHours(1); // Backoff when refresh fails to ensure ZTS is not DDoS'ed. - private static final Path CONTAINER_SIA_DIRECTORY = Paths.get("/var/lib/sia"); + private static final String CONTAINER_SIA_DIRECTORY = "/var/lib/sia"; private final URI ztsEndpoint; private final Path trustStorePath; private final AthenzIdentity configserverIdentity; private final Clock clock; private final ServiceIdentityProvider hostIdentityProvider; - private final FileSystem fileSystem; private final IdentityDocumentClient identityDocumentClient; private final CsrGenerator csrGenerator; private final boolean useInternalZts; @@ -83,7 +82,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { String certificateDnsSuffix, ServiceIdentityProvider hostIdentityProvider, boolean useInternalZts) { - this(ztsEndpoint, trustStorePath, configServerInfo, certificateDnsSuffix, hostIdentityProvider, useInternalZts, Clock.systemUTC(), FileSystems.getDefault()); + this(ztsEndpoint, trustStorePath, configServerInfo, certificateDnsSuffix, hostIdentityProvider, useInternalZts, Clock.systemUTC()); } public AthenzCredentialsMaintainer(URI ztsEndpoint, @@ -92,14 +91,12 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { String certificateDnsSuffix, ServiceIdentityProvider hostIdentityProvider, boolean useInternalZts, - Clock clock, - FileSystem fileSystem) { + Clock clock) { this.ztsEndpoint = ztsEndpoint; this.trustStorePath = trustStorePath; this.configserverIdentity = configServerInfo.getConfigServerIdentity(); this.csrGenerator = new CsrGenerator(certificateDnsSuffix, configserverIdentity.getFullName()); this.hostIdentityProvider = hostIdentityProvider; - this.fileSystem = fileSystem; this.identityDocumentClient = new DefaultIdentityDocumentClient( configServerInfo.getLoadBalancerEndpoint(), hostIdentityProvider, @@ -164,7 +161,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { @Override public Duration certificateLifetime(NodeAgentContext context) { - Path containerSiaDirectory = fileSystem.getPath(context.pathOnHostFromPathInNode(CONTAINER_SIA_DIRECTORY).toString()); + Path containerSiaDirectory = context.pathOnHostFromPathInNode(CONTAINER_SIA_DIRECTORY); Path certificateFile = SiaUtils.getCertificateFile(containerSiaDirectory, context.identity()); try { X509Certificate certificate = readCertificateFromFile(certificateFile); @@ -266,11 +263,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } private static void writeFile(Path path, int vespaUidOnHost, String utf8Content) { - new UnixPath(path.toString() + ".tmp") - .deleteIfExists() - .createNewFile("r--------") + new UnixPath(path.resolveSibling(path.getFileName() + ".tmp")) + .writeUtf8File(utf8Content, "r--------") .setOwnerId(vespaUidOnHost) - .writeUtf8File(utf8Content) .atomicMove(path); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java index 23a6ed2aa8c..27b3467163c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java @@ -100,8 +100,7 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { directoryOnHost.deleteRecursively(); } context.log(log, Level.INFO, "Creating '" + directoryOnHost +"'."); - directoryOnHost.createDirectory(); - directoryOnHost.setPermissions("rwxrwxrwx"); + directoryOnHost.createDirectory("rwxrwxrwx"); URI destination = serviceDumpDestination(nodeSpec, createDumpId(request)); ProducerContext producerCtx = new ProducerContext(context, directoryInNode.toPath(), request); List<Artifact> producedArtifacts = new ArrayList<>(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 2751d4d96b0..6925c1e9600 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -204,6 +204,19 @@ public class NodeAgentContextImpl implements NodeAgentContext { return path; } + public static NodeAgentContextImpl.Builder builder(NodeSpec node) { + return new Builder(new NodeSpec.Builder(node)); + } + + /** + * Creates a NodeAgentContext.Builder with a NodeSpec that has the given hostname and some + * reasonable values for the remaining required NodeSpec fields. Use {@link #builder(NodeSpec)} + * if you want to control the entire NodeSpec. + */ + public static NodeAgentContextImpl.Builder builder(String hostname) { + return new Builder(NodeSpec.Builder.testSpec(hostname)); + } + /** For testing only! */ public static class Builder { private NodeSpec.Builder nodeSpecBuilder; @@ -218,17 +231,8 @@ public class NodeAgentContextImpl implements NodeAgentContext { private Path containerStorage; private Optional<ApplicationId> hostExclusiveTo = Optional.empty(); - public Builder(NodeSpec node) { - this.nodeSpecBuilder = new NodeSpec.Builder(node); - } - - /** - * Creates a NodeAgentContext.Builder with a NodeSpec that has the given hostname and some - * reasonable values for the remaining required NodeSpec fields. Use {@link #Builder(NodeSpec)} - * if you want to control the entire NodeSpec. - */ - public Builder(String hostname) { - this.nodeSpecBuilder = NodeSpec.Builder.testSpec(hostname); + private Builder(NodeSpec.Builder nodeSpecBuilder) { + this.nodeSpecBuilder = nodeSpecBuilder; } public Builder nodeSpecBuilder(Function<NodeSpec.Builder, NodeSpec.Builder> nodeSpecBuilderModifier) { @@ -316,7 +320,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { }), fileSystem, Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new), - Optional.ofNullable(containerStorage).orElseGet(() -> fileSystem.getPath("/home/docker/container-storage")), + Optional.ofNullable(containerStorage).orElseGet(() -> fileSystem.getPath("/data/vespa/storage")), fileSystem.getPath("/opt/vespa"), Optional.ofNullable(userNamespace).orElseGet(() -> new UserNamespace(10000, 10000, "vespa", "users", 1000, 100)), cpuSpeedUp, hostExclusiveTo); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java index 020a8b6c7f6..5b53879ceec 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/UserNamespace.java @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; /** diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 2ce49ae383a..6949f648865 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; @@ -12,6 +14,7 @@ import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.FileTime; import java.nio.file.attribute.GroupPrincipal; @@ -35,6 +38,10 @@ import static com.yahoo.yolean.Exceptions.uncheck; */ // @Immutable public class UnixPath { + + private static final Set<OpenOption> DEFAULT_OPEN_OPTIONS = + Set.of(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + private final Path path; public UnixPath(Path path) { this.path = path; } @@ -97,13 +104,29 @@ public class UnixPath { return writeBytes(content.getBytes(StandardCharsets.UTF_8), options); } + public UnixPath writeUtf8File(String content, String permissions, OpenOption... options) { + return writeBytes(content.getBytes(StandardCharsets.UTF_8), permissions, options); + } + public UnixPath writeBytes(byte[] content, OpenOption... options) { - uncheck(() -> Files.write(path, content, options)); - return this; + return writeBytes(content, null, options); } - public UnixPath atomicWriteUt8(String content) { - return atomicWriteBytes(content.getBytes(StandardCharsets.UTF_8)); + public UnixPath writeBytes(byte[] content, String permissions, OpenOption... options) { + FileAttribute<?>[] attributes = Optional.ofNullable(permissions) + .map(this::getPosixFilePermissionsFromString) + .map(PosixFilePermissions::asFileAttribute) + .map(attribute -> new FileAttribute<?>[]{attribute}) + .orElseGet(() -> new FileAttribute<?>[0]); + + Set<OpenOption> optionsSet = options.length == 0 ? DEFAULT_OPEN_OPTIONS : Set.of(options); + + try (SeekableByteChannel channel = Files.newByteChannel(path, optionsSet, attributes)) { + channel.write(ByteBuffer.wrap(content)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return this; } /** Write a file to the same dir as this, and then atomically move it to this' path. */ @@ -184,18 +207,8 @@ public class UnixPath { return this; } - public UnixPath createNewFile(String permissions) { - FileAttribute<?> attribute = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(permissions)); - uncheck(() -> Files.createFile(path, attribute)); - return this; - } - public UnixPath createParents() { - Path parent = path.getParent(); - if (!Files.isDirectory(parent)) { - uncheck(() -> Files.createDirectories(parent)); - } - + uncheck(() -> Files.createDirectories(path.getParent())); return this; } @@ -275,7 +288,7 @@ public class UnixPath { } catch (NoSuchFileException ignored) { return Stream.empty(); } catch (IOException e) { - throw new RuntimeException("Failed to list contents of directory " + path.toAbsolutePath(), e); + throw new UncheckedIOException("Failed to list contents of directory " + path.toAbsolutePath(), e); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java index 393137f795e..495b72e4554 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystem.java @@ -17,7 +17,7 @@ public class ContainerFileSystem extends FileSystem { private final ContainerFileSystemProvider containerFsProvider; - public ContainerFileSystem(ContainerFileSystemProvider containerFsProvider) { + ContainerFileSystem(ContainerFileSystemProvider containerFsProvider) { this.containerFsProvider = containerFsProvider; } @@ -81,4 +81,7 @@ public class ContainerFileSystem extends FileSystem { throw new UnsupportedOperationException(); } + public static ContainerFileSystem create(Path containerStorageRoot, int uidOffset, int gidOffset) { + return new ContainerFileSystemProvider(containerStorageRoot, uidOffset, gidOffset).getFileSystem(null); + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java index 79214334c55..73cd3f8cfc5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java @@ -37,14 +37,12 @@ import static com.yahoo.yolean.Exceptions.uncheck; /** * @author valerijf */ -public class ContainerFileSystemProvider extends FileSystemProvider { - +class ContainerFileSystemProvider extends FileSystemProvider { private final ContainerFileSystem containerFs; private final ContainerUserPrincipalLookupService userPrincipalLookupService; private final Path containerRootOnHost; - - public ContainerFileSystemProvider(Path containerRootOnHost, int uidOffset, int gidOffset) { + ContainerFileSystemProvider(Path containerRootOnHost, int uidOffset, int gidOffset) { this.containerFs = new ContainerFileSystem(this); this.userPrincipalLookupService = new ContainerUserPrincipalLookupService( containerRootOnHost.getFileSystem().getUserPrincipalLookupService(), uidOffset, gidOffset); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java index e967806dc55..15295ffd087 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.task.util.fs; import java.io.IOException; import java.net.URI; -import java.nio.file.FileSystem; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.WatchEvent; @@ -17,6 +16,8 @@ import java.util.Objects; import static com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerFileSystemProvider.toContainerPath; /** + * Represents a path in container that is mapped in from the host. ContainerPaths are always normalized and absolute. + * * @author valerijf */ public class ContainerPath implements Path { @@ -36,12 +37,11 @@ public class ContainerPath implements Path { throw new IllegalArgumentException("Path on host (" + pathOnHost + ") must start with container root on host (" + containerRootOnHost + ")"); } - public Path pathOnHost() { - return pathOnHost; - } + public Path pathOnHost() { return pathOnHost; } + public String pathInContainer() { return '/' + String.join("/", parts); } @Override - public FileSystem getFileSystem() { + public ContainerFileSystem getFileSystem() { return containerFs; } @@ -83,15 +83,9 @@ public class ContainerPath implements Path { return Path.of(parts[beginIndex], rest); } - @Override - public ContainerPath resolve(Path other) { - return resolve(containerFs, parts, other); - } - - @Override - public ContainerPath resolveSibling(String other) { - return resolve(Path.of("..", other)); - } + @Override public ContainerPath resolve(Path other) { return resolve(containerFs, parts, other); } + @Override public ContainerPath resolve(String other) { return resolve(Path.of(other)); } + @Override public ContainerPath resolveSibling(String other) { return resolve(Path.of("..", other)); } @Override public boolean startsWith(Path other) { @@ -179,7 +173,7 @@ public class ContainerPath implements Path { @Override public String toString() { - return '/' + String.join("/", parts); + return containerFs.provider().containerRootOnHost().getFileName() + ":" + pathInContainer(); } private static ContainerPath resolve(ContainerFileSystem containerFs, String[] currentParts, Path other) { @@ -199,25 +193,25 @@ public class ContainerPath implements Path { parts.toArray(String[]::new)); } - static ContainerPath fromPathInContainer(ContainerFileSystem containerFs, Path pathInContainer) { + public static ContainerPath fromPathInContainer(ContainerFileSystem containerFs, Path pathInContainer) { if (!pathInContainer.isAbsolute()) throw new IllegalArgumentException("Path in container must be absolute: " + pathInContainer); return resolve(containerFs, new String[0], pathInContainer); } -static ContainerPath fromPathOnHost(ContainerFileSystem containerFs, Path pathOnHost) { - pathOnHost = pathOnHost.normalize(); - Path containerRootOnHost = containerFs.provider().containerRootOnHost(); - Path pathUnderContainerStorage = containerRootOnHost.relativize(pathOnHost); + public static ContainerPath fromPathOnHost(ContainerFileSystem containerFs, Path pathOnHost) { + pathOnHost = pathOnHost.normalize(); + Path containerRootOnHost = containerFs.provider().containerRootOnHost(); + Path pathUnderContainerStorage = containerRootOnHost.relativize(pathOnHost); - if (pathUnderContainerStorage.getNameCount() == 0 || pathUnderContainerStorage.getName(0).toString().isEmpty()) - return new ContainerPath(containerFs, pathOnHost, new String[0]); - if (pathUnderContainerStorage.getName(0).toString().equals("..")) - throw new IllegalArgumentException("Path " + pathOnHost + " is not under container root " + containerRootOnHost); + if (pathUnderContainerStorage.getNameCount() == 0 || pathUnderContainerStorage.getName(0).toString().isEmpty()) + return new ContainerPath(containerFs, pathOnHost, new String[0]); + if (pathUnderContainerStorage.getName(0).toString().equals("..")) + throw new IllegalArgumentException("Path " + pathOnHost + " is not under container root " + containerRootOnHost); - List<String> parts = new ArrayList<>(); - for (int i = 0; i < pathUnderContainerStorage.getNameCount(); i++) - parts.add(pathUnderContainerStorage.getName(i).toString()); - return new ContainerPath(containerFs, pathOnHost, parts.toArray(String[]::new)); -} + List<String> parts = new ArrayList<>(); + for (int i = 0; i < pathUnderContainerStorage.getNameCount(); i++) + parts.add(pathUnderContainerStorage.getName(i).toString()); + return new ContainerPath(containerFs, pathOnHost, parts.toArray(String[]::new)); + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java index 80dbbfb9d13..82fb63b0036 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java @@ -1,13 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.container; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; -import java.io.IOException; import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Map; import java.util.Optional; @@ -22,7 +20,7 @@ public class ContainerStatsCollectorTest { private final FileSystem fileSystem = TestFileSystem.create(); @Test - public void collect() throws Exception { + public void collect() { ContainerStatsCollector collector = new ContainerStatsCollector(fileSystem); ContainerId containerId = new ContainerId("id1"); int containerPid = 42; @@ -44,22 +42,21 @@ public class ContainerStatsCollectorTest { stats.get().getNetworks()); } - private void mockNetworkStats(int pid) throws IOException { - Path dev = fileSystem.getPath("/proc/" + pid + "/net/dev"); - Files.createDirectories(dev.getParent()); - Files.writeString(dev, "Inter-| Receive | Transmit\n" + + private void mockNetworkStats(int pid) { + UnixPath dev = new UnixPath(fileSystem.getPath("/proc/" + pid + "/net/dev")); + dev.createParents().writeUtf8File("Inter-| Receive | Transmit\n" + " face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" + " lo: 36289258 149700 0 0 0 0 0 0 36289258 149700 0 0 0 0 0 0\n" + " eth0: 22280813 118083 3 4 0 0 0 0 19859383 115415 5 6 0 0 0 0\n"); } - private void mockMemoryStats(ContainerId containerId) throws IOException { - Path root = fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-" + containerId + ".scope"); - Files.createDirectories(root); + private void mockMemoryStats(ContainerId containerId) { + UnixPath root = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-" + containerId + ".scope")); + root.createDirectories(); - Files.writeString(root.resolve("memory.limit_in_bytes"), "2147483648\n"); - Files.writeString(root.resolve("memory.usage_in_bytes"), "1228017664\n"); - Files.writeString(root.resolve("memory.stat"), "cache 470790144\n" + + root.resolve("memory.limit_in_bytes").writeUtf8File("2147483648\n"); + root.resolve("memory.usage_in_bytes").writeUtf8File("1228017664\n"); + root.resolve("memory.stat").writeUtf8File("cache 470790144\n" + "rss 698699776\n" + "rss_huge 526385152\n" + "shmem 0\n" + @@ -97,24 +94,25 @@ public class ContainerStatsCollectorTest { "total_unevictable 0\n"); } - private void mockCpuStats(ContainerId containerId) throws IOException { - Path root = fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-" + containerId + ".scope"); - Path proc = fileSystem.getPath("/proc"); - Files.createDirectories(root); - Files.createDirectories(proc); - Files.writeString(root.resolve("cpu.stat"), "nr_periods 1\n" + + private void mockCpuStats(ContainerId containerId) { + UnixPath root = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-" + containerId + ".scope")); + UnixPath proc = new UnixPath(fileSystem.getPath("/proc")); + root.createDirectories(); + proc.createDirectories(); + + root.resolve("cpu.stat").writeUtf8File("nr_periods 1\n" + "nr_throttled 2\n" + "throttled_time 3\n"); - Files.writeString(root.resolve("cpuacct.usage_percpu"), "25801608855 22529436415 25293652376 26212081533 " + + root.resolve("cpuacct.usage_percpu").writeUtf8File("25801608855 22529436415 25293652376 26212081533 " + "27545883290 25357818592 33464821448 32568003867 " + "28916742231 31771772292 34418037242 38417072233 " + "26069101127 24568838237 23683334366 26824607997 " + "24289870206 22249389818 32683986446 32444831154 " + "30488394217 26840956322 31633747261 30838696584\n"); - Files.writeString(root.resolve("cpuacct.usage"), "691675615472\n"); - Files.writeString(root.resolve("cpuacct.stat"), "user 40900\n" + + root.resolve("cpuacct.usage").writeUtf8File("691675615472\n"); + root.resolve("cpuacct.stat").writeUtf8File("user 40900\n" + "system 26219\n"); - Files.writeString(proc.resolve("stat"), "cpu 7991366 978222 2346238 565556517 1935450 25514479 615206 0 0 0\n" + + proc.resolve("stat").writeUtf8File("cpu 7991366 978222 2346238 565556517 1935450 25514479 615206 0 0 0\n" + "cpu0 387906 61529 99088 23516506 42258 1063359 29882 0 0 0\n" + "cpu1 271253 49383 86149 23655234 41703 1061416 31885 0 0 0\n" + "cpu2 349420 50987 93560 23571695 59437 1051977 24461 0 0 0\n" + diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java index b3404b4108e..8683c2cb417 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java @@ -33,7 +33,7 @@ public class ContainerFailTest { .build(); tester.addChildNodeRepositoryNode(nodeSpec); - NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec).build(); + NodeAgentContext context = NodeAgentContextImpl.builder(nodeSpec).build(); tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any(), any()); tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java index 38c1dcb2fb6..08e335f188a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java @@ -91,7 +91,7 @@ public class ContainerTester implements AutoCloseable { VespaServiceDumper.DUMMY_INSTANCE); nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, clock, Duration.ofMillis(10), Duration.ZERO); NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> - new NodeAgentContextImpl.Builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); + NodeAgentContextImpl.builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, nodeAdmin, HOST_HOSTNAME, clock, flagSource); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 241f04ee2d6..b7b247bbf87 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -47,14 +47,14 @@ public class StorageMaintainerTest { private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(1234567890)); private final FileSystem fileSystem = TestFileSystem.create(); private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, clock, - fileSystem.getPath("/home/docker/container-storage/container-archive")); + fileSystem.getPath("/data/vespa/storage/container-archive")); @Test public void testDiskUsed() throws IOException { - NodeAgentContext context = new NodeAgentContextImpl.Builder("host-1.domain.tld").fileSystem(fileSystem).build(); + NodeAgentContext context = NodeAgentContextImpl.builder("host-1.domain.tld").fileSystem(fileSystem).build(); Files.createDirectories(context.pathOnHostFromPathInNode("/")); - terminal.expectCommand("du -xsk /home/docker/container-storage/host-1 2>&1", 0, "321\t/home/docker/container-storage/host-1/"); + terminal.expectCommand("du -xsk /data/vespa/storage/host-1 2>&1", 0, "321\t/data/vespa/storage/host-1/"); assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); // Value should still be cached, no new execution against the terminal @@ -73,7 +73,7 @@ public class StorageMaintainerTest { NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1"); createNodeAgentContextAndContainerStorage(fileSystem, "container-2"); - Path pathToArchiveDir = fileSystem.getPath("/home/docker/container-storage/container-archive"); + Path pathToArchiveDir = fileSystem.getPath("/data/vespa/storage/container-archive"); Files.createDirectories(pathToArchiveDir); Path containerStorageRoot = context1.pathOnHostFromPathInNode("/").getParent(); @@ -112,7 +112,7 @@ public class StorageMaintainerTest { } private static NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { - NodeAgentContext context = new NodeAgentContextImpl.Builder(containerName + ".domain.tld") + NodeAgentContext context = NodeAgentContextImpl.builder(containerName + ".domain.tld") .fileSystem(fileSystem).build(); Path containerVespaHomeOnHost = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("")); @@ -142,7 +142,7 @@ public class StorageMaintainerTest { @Test public void not_run_if_not_enough_used() throws IOException { - NodeAgentContext context = new NodeAgentContextImpl.Builder( + NodeAgentContext context = NodeAgentContextImpl.builder( NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) .fileSystem(fileSystem).build(); Files.createDirectories(context.pathOnHostFromPathInNode("/")); @@ -154,7 +154,7 @@ public class StorageMaintainerTest { @Test public void deletes_correct_amount() throws IOException { - NodeAgentContext context = new NodeAgentContextImpl.Builder( + NodeAgentContext context = NodeAgentContextImpl.builder( NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) .fileSystem(fileSystem).build(); @@ -172,6 +172,6 @@ public class StorageMaintainerTest { } private void mockDiskUsage(long kBytes) { - terminal.expectCommand("du -xsk /home/docker/container-storage/h123a 2>&1", 0, kBytes + "\t/path"); + terminal.expectCommand("du -xsk /data/vespa/storage/h123a 2>&1", 0, kBytes + "\t/path"); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index a10ce1c7d74..80fde82a89f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -41,7 +41,7 @@ public class AclMaintainerTest { private final FileSystem fileSystem = TestFileSystem.create(); private final Function<Acl, NodeAgentContext> contextGenerator = - acl -> new NodeAgentContextImpl.Builder("container1.host.com").fileSystem(fileSystem).acl(acl).build(); + acl -> NodeAgentContextImpl.builder("container1.host.com").fileSystem(fileSystem).acl(acl).build(); private final List<String> writtenFileContents = new ArrayList<>(); @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index f56794382e8..eacaa038194 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import org.junit.Test; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.List; import java.util.Map; @@ -28,10 +27,10 @@ public class CoreCollectorTest { private final String JDK_PATH = "/path/to/jdk/java"; private final ContainerOperations docker = mock(ContainerOperations.class); private final CoreCollector coreCollector = new CoreCollector(docker); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); + private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld").build(); - private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234"); - private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program"); + private final Path TEST_CORE_PATH = Path.of("/tmp/core.1234"); + private final String TEST_BIN_PATH = "/usr/bin/program"; private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]", "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", "#0 0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t printf(argv[3]);", @@ -53,7 +52,7 @@ public class CoreCollectorTest { mockExec(cmd, "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin//program'"); + "'/usr/bin/program'"); assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(context, TEST_CORE_PATH)); mockExec(cmd, @@ -64,7 +63,7 @@ public class CoreCollectorTest { mockExec(new String[]{"stat", GDB_PATH_RHEL7_DT9}, "The stat output"); - Path fallbackResponse = Paths.get("/response/from/fallback"); + String fallbackResponse = "/response/from/fallback"; mockExec(new String[]{"/bin/sh", "-c", GDB_PATH_RHEL7_DT9 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}, "Core was generated by `/response/from/fallback'."); mockExec(cmd, @@ -84,7 +83,7 @@ public class CoreCollectorTest { GDB_PATH_RHEL7_DT10 + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}; mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'."); - assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(context, TEST_CORE_PATH)); + assertEquals("/usr/bin/program-from-gdb", coreCollector.readBinPathFallback(context, TEST_CORE_PATH)); mockExec(cmd, "", "Error 123"); try { @@ -139,7 +138,7 @@ public class CoreCollectorTest { String.join("\n", GDB_BACKTRACE)); Map<String, Object> expectedData = Map.of( - "bin_path", TEST_BIN_PATH.toString(), + "bin_path", TEST_BIN_PATH, "backtrace", GDB_BACKTRACE, "backtrace_all_threads", GDB_BACKTRACE); assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH)); @@ -154,7 +153,7 @@ public class CoreCollectorTest { mockExec(new String[]{GDB_PATH_RHEL7_DT9 + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"}, "", "Failure"); - Map<String, Object> expectedData = Map.of("bin_path", TEST_BIN_PATH.toString()); + Map<String, Object> expectedData = Map.of("bin_path", TEST_BIN_PATH); assertEquals(expectedData, coreCollector.collect(context, TEST_CORE_PATH)); } @@ -181,7 +180,7 @@ public class CoreCollectorTest { @Test public void metadata_for_java_heap_dump() { - assertEquals(JAVA_HEAP_DUMP_METADATA, coreCollector.collect(context, Paths.get("dump_java_pid123.hprof"))); + assertEquals(JAVA_HEAP_DUMP_METADATA, coreCollector.collect(context, Path.of("dump_java_pid123.hprof"))); } private void mockExec(String[] cmd, String output) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 673ad9eee86..31883311e33 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -45,7 +45,7 @@ import static org.mockito.Mockito.when; */ public class CoredumpHandlerTest { private final FileSystem fileSystem = TestFileSystem.create(); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld") + private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") .fileSystem(fileSystem).build(); private final Path crashPathInContainer = fileSystem.getPath("/var/crash"); private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); @@ -58,7 +58,7 @@ public class CoredumpHandlerTest { @SuppressWarnings("unchecked") private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); private final CoredumpHandler coredumpHandler = new CoredumpHandler(terminal, coreCollector, coredumpReporter, - crashPathInContainer, doneCoredumpsPath, 100, metrics, clock, coredumpIdSupplier); + crashPathInContainer.toString(), doneCoredumpsPath, 100, metrics, clock, coredumpIdSupplier); @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java index 0853223d142..68127231554 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java @@ -40,8 +40,8 @@ class VespaServiceDumperImplTest { private static final String HOSTNAME = "host-1.domain.tld"; - private final FileSystem fileSystem = TestFileSystem.create();; - private final Path tmpDirectory = fileSystem.getPath("/home/docker/container-storage/host-1/opt/vespa/tmp"); + private final FileSystem fileSystem = TestFileSystem.create(); + private final Path tmpDirectory = fileSystem.getPath("/data/vespa/storage/host-1/opt/vespa/tmp"); @BeforeEach void create_tmp_directory() throws IOException { @@ -73,7 +73,7 @@ class VespaServiceDumperImplTest { VespaServiceDumper reporter = new VespaServiceDumperImpl( ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, clock); - NodeAgentContextImpl context = new NodeAgentContextImpl.Builder(nodeSpec) + NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) .fileSystem(fileSystem) .build(); reporter.processServiceDumpRequest(context); @@ -115,7 +115,7 @@ class VespaServiceDumperImplTest { VespaServiceDumper reporter = new VespaServiceDumperImpl( ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, clock); - NodeAgentContextImpl context = new NodeAgentContextImpl.Builder(nodeSpec) + NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) .fileSystem(fileSystem) .build(); reporter.processServiceDumpRequest(context); @@ -158,7 +158,7 @@ class VespaServiceDumperImplTest { new ServiceDumpReport.DumpOptions(true, 20.0, null)); VespaServiceDumper reporter = new VespaServiceDumperImpl( ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, clock); - NodeAgentContextImpl context = new NodeAgentContextImpl.Builder(nodeSpec) + NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) .fileSystem(fileSystem) .build(); reporter.processServiceDumpRequest(context); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index ed400a160c9..232b95f5ede 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -154,7 +154,7 @@ public class NodeAdminImplTest { } private NodeAgentContext createNodeAgentContext(String hostname) { - return new NodeAgentContextImpl.Builder(hostname).build(); + return NodeAgentContextImpl.builder(hostname).build(); } private NodeAgentWithScheduler mockNodeAgentWithSchedulerFactory(NodeAgentContext context) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java index 680d197b77d..b91afd0bfef 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java @@ -18,17 +18,17 @@ import static org.junit.Assert.assertTrue; */ public class NodeAgentContextImplTest { private final FileSystem fileSystem = TestFileSystem.create(); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-1.domain.tld") + private final NodeAgentContext context = NodeAgentContextImpl.builder("container-1.domain.tld") .fileSystem(fileSystem).build(); @Test public void path_on_host_from_path_in_node_test() { assertEquals( - "/home/docker/container-storage/container-1", + "/data/vespa/storage/container-1", context.pathOnHostFromPathInNode("/").toString()); assertEquals( - "/home/docker/container-storage/container-1/dev/null", + "/data/vespa/storage/container-1/dev/null", context.pathOnHostFromPathInNode("/dev/null").toString()); } @@ -41,7 +41,7 @@ public class NodeAgentContextImplTest { public void path_in_node_from_path_on_host_test() { assertEquals( "/dev/null", - context.pathInNodeFromPathOnHost(fileSystem.getPath("/home/docker/container-storage/container-1/dev/null")).toString()); + context.pathInNodeFromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-1/dev/null")).toString()); } @Test(expected=IllegalArgumentException.class) @@ -51,7 +51,7 @@ public class NodeAgentContextImplTest { @Test(expected=IllegalArgumentException.class) public void path_on_host_must_be_inside_container_storage_of_context() { - context.pathInNodeFromPathOnHost(fileSystem.getPath("/home/docker/container-storage/container-2/dev/null")); + context.pathInNodeFromPathOnHost(fileSystem.getPath("/data/vespa/storage/container-2/dev/null")); } @Test(expected=IllegalArgumentException.class) @@ -89,6 +89,6 @@ public class NodeAgentContextImplTest { private static NodeAgentContext createContextWithDisabledTasks(String... tasks) { InMemoryFlagSource flagSource = new InMemoryFlagSource(); flagSource.withListFlag(PermanentFlags.DISABLED_HOST_ADMIN_TASKS.id(), List.of(tasks), String.class); - return new NodeAgentContextImpl.Builder("node123").flagSource(flagSource).build(); + return NodeAgentContextImpl.builder("node123").flagSource(flagSource).build(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java index 2ed3b66c1b1..51fedd54381 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -142,7 +142,7 @@ public class NodeAgentContextManagerTest { } private static NodeAgentContext generateContext() { - return new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); + return NodeAgentContextImpl.builder("container-123.domain.tld").build(); } private static class AsyncExecutor<T> { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 3aec6c1902a..8764db502bb 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -233,7 +233,7 @@ public class NodeAgentImplTest { nodeAgent.doConverge(secondContext); inOrder.verify(orchestrator, never()).resume(any(String.class)); - NodeAgentContext thirdContext = new NodeAgentContextImpl.Builder(specBuilder.vcpu(5).build()).cpuSpeedUp(1.25).build(); + NodeAgentContext thirdContext = NodeAgentContextImpl.builder(specBuilder.vcpu(5).build()).cpuSpeedUp(1.25).build(); nodeAgent.doConverge(thirdContext); ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16); mockGetContainer(dockerImage, resourcesAfterThird, true); @@ -777,7 +777,7 @@ public class NodeAgentImplTest { } private NodeAgentContext createContext(NodeSpec nodeSpec) { - return new NodeAgentContextImpl.Builder(nodeSpec).build(); + return NodeAgentContextImpl.builder(nodeSpec).build(); } private NodeSpec.Builder nodeBuilder(NodeState state) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java index 80e88e302d3..0bb820bbfdc 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java @@ -26,7 +26,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.google.common.collect.ImmutableSet.of; +import static java.util.Set.of; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java index 952bf04620f..01c7e9265ac 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.ComparisonFailure; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; @@ -147,7 +148,7 @@ public class UnixPathTest { var path = new UnixPath(fs.getPath("/dir/foo")); path.createParents(); path.writeUtf8File("bar"); - path.atomicWriteUt8("bar v2"); + path.atomicWriteBytes("bar v2".getBytes(StandardCharsets.UTF_8)); assertEquals("bar v2", path.readUtf8File()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java index 38c1e2720c3..eb15450b756 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java @@ -21,7 +21,7 @@ class ContainerFileSystemTest { private final FileSystem fileSystem = TestFileSystem.create(); private final UnixPath containerRootOnHost = new UnixPath(fileSystem.getPath("/data/storage/ctr1")); - private final ContainerFileSystem containerFs = new ContainerFileSystemProvider(containerRootOnHost.createDirectories().toPath(), 10_000, 11_000).getFileSystem(null); + private final ContainerFileSystem containerFs = ContainerFileSystem.create(containerRootOnHost.createDirectories().toPath(), 10_000, 11_000); @Test public void creates_files_and_directories_with_container_root_as_owner() throws IOException { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java index ead4ad7ecde..ebbbaf3b525 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPathTest.java @@ -25,7 +25,7 @@ import java.nio.file.Path; class ContainerPathTest { private final FileSystem baseFs = TestFileSystem.create(); - private final ContainerFileSystem containerFs = new ContainerFileSystemProvider(baseFs.getPath("/data/storage/ctr1"), 0, 0).getFileSystem(null); + private final ContainerFileSystem containerFs = ContainerFileSystem.create(baseFs.getPath("/data/storage/ctr1"), 0, 0); @Test public void create_new_container_path() { @@ -62,7 +62,7 @@ class ContainerPathTest { assertFalse(parent.startsWith(path)); assertFalse(path.startsWith(Path.of(path.toString()))); - assertTrue(path.endsWith(Path.of(path.toString()))); + assertTrue(path.endsWith(Path.of(path.pathInContainer()))); assertTrue(path.endsWith(Path.of("logs/file"))); assertFalse(path.endsWith(Path.of("/logs/file"))); } @@ -102,7 +102,7 @@ class ContainerPathTest { private static void assertPaths(ContainerPath actual, String expectedPathOnHost, String expectedPathInContainer) { assertEquals(expectedPathOnHost, actual.pathOnHost().toString()); - assertEquals(expectedPathInContainer, actual.toString()); + assertEquals(expectedPathInContainer, actual.pathInContainer()); } private static void assertThrows(Executable executable, String expectedMsg) { |