summaryrefslogtreecommitdiffstats
path: root/node-admin
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2018-10-23 20:42:58 +0200
committerJon Bratseth <bratseth@oath.com>2018-10-23 20:42:58 +0200
commit098f7fd34dfb3cdbb7f05c0bfe635c2f39cea2b3 (patch)
tree1bd87252dfb71c39e1dc448349a22579eefe06f2 /node-admin
parente678ffc025674df331f478064542db22fd7e4dd5 (diff)
parent1a8bbf7434e13271f10862aa3c0057f30122ad0c (diff)
Merge with master
Diffstat (limited to 'node-admin')
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java8
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java318
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java83
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java14
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java223
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java177
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java4
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java2
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java2
-rw-r--r--node-admin/src/main/resources/configdefinitions/config-server.def8
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java29
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java129
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java324
14 files changed, 11 insertions, 1315 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
index 7b484dfc481..ec911cc5600 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
@@ -2,8 +2,6 @@
package com.yahoo.vespa.hosted.node.admin.component;
import com.yahoo.vespa.athenz.api.AthenzService;
-import com.yahoo.vespa.athenz.utils.AthenzIdentities;
-import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
import java.net.URI;
import java.util.ArrayList;
@@ -24,12 +22,6 @@ public class ConfigServerInfo {
private final Map<String, URI> configServerURIs;
private final AthenzService configServerIdentity;
- // TODO: Remove
- public ConfigServerInfo(ConfigServerConfig config) {
- this(config.loadBalancerHost(), config.hosts(), config.scheme(), config.port(),
- (AthenzService) AthenzIdentities.from(config.configserverAthenzIdentity()));
- }
-
public ConfigServerInfo(String loadBalancerHostName, List<String> configServerHostNames,
String scheme, int port, AthenzService configServerAthenzIdentity) {
this.configServerHostNames = configServerHostNames;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
deleted file mode 100644
index aaadd3cb24e..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
+++ /dev/null
@@ -1,318 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.component;
-
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.athenz.api.AthenzService;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
-import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
-import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses;
-import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesImpl;
-
-import java.net.URI;
-import java.nio.file.Path;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-
-/**
- * Various utilities for getting values from node-admin's environment. Immutable.
- *
- * @author Øyvind Bakksjø
- * @author hmusum
- */
-public class Environment {
- private final ConfigServerInfo configServerInfo;
- private final String environment;
- private final String region;
- private final String system;
- private final String cloud;
- private final String parentHostHostname;
- private final IPAddresses ipAddresses;
- private final PathResolver pathResolver;
- private final List<String> logstashNodes;
- private final NodeType nodeType;
- private final ContainerEnvironmentResolver containerEnvironmentResolver;
- private final String certificateDnsSuffix;
- private final URI ztsUri;
- private final AthenzService nodeAthenzIdentity;
- private final boolean nodeAgentCertEnabled;
- private final Path trustStorePath;
- private final DockerNetworking dockerNetworking;
-
- private Environment(ConfigServerInfo configServerInfo,
- Path trustStorePath,
- String environment,
- String region,
- String system,
- String cloud,
- String parentHostHostname,
- IPAddresses ipAddresses,
- PathResolver pathResolver,
- List<String> logstashNodes,
- NodeType nodeType,
- ContainerEnvironmentResolver containerEnvironmentResolver,
- String certificateDnsSuffix,
- URI ztsUri,
- AthenzService nodeAthenzIdentity,
- boolean nodeAgentCertEnabled,
- DockerNetworking dockerNetworking) {
- this.configServerInfo = Objects.requireNonNull(configServerInfo, "configServerConfig cannot be null");
- this.environment = Objects.requireNonNull(environment, "environment cannot be null");;
- this.region = Objects.requireNonNull(region, "region cannot be null");;
- this.system = Objects.requireNonNull(system, "system cannot be null");;
- this.cloud = Objects.requireNonNull(cloud, "cloud cannot be null");
- this.parentHostHostname = parentHostHostname;
- this.ipAddresses = ipAddresses;
- this.pathResolver = pathResolver;
- this.logstashNodes = logstashNodes;
- this.nodeType = nodeType;
- this.containerEnvironmentResolver = containerEnvironmentResolver;
- this.certificateDnsSuffix = certificateDnsSuffix;
- this.ztsUri = ztsUri;
- this.nodeAthenzIdentity = nodeAthenzIdentity;
- this.nodeAgentCertEnabled = nodeAgentCertEnabled;
- this.trustStorePath = trustStorePath;
- this.dockerNetworking = Objects.requireNonNull(dockerNetworking, "dockerNetworking cannot be null");
- }
-
- public List<String> getConfigServerHostNames() { return configServerInfo.getConfigServerHostNames(); }
-
- public String getEnvironment() { return environment; }
-
- public String getRegion() {
- return region;
- }
-
- public String getSystem() {
- return system;
- }
-
- public String getCloud() { return cloud; }
-
- public String getParentHostHostname() {
- return parentHostHostname;
- }
-
- public String getZone() {
- return getEnvironment() + "." + getRegion();
- }
-
- public IPAddresses getIpAddresses() {
- return ipAddresses;
- }
-
- public PathResolver getPathResolver() {
- return pathResolver;
- }
-
- /**
- * Translates an absolute path in node agent container to an absolute path in node admin container.
- * @param containerName name of the node agent container
- * @param pathInNode absolute path in that container
- * @return the absolute path in node admin container pointing at the same inode
- */
- public Path pathInNodeAdminFromPathInNode(ContainerName containerName, Path pathInNode) {
- if (! pathInNode.isAbsolute()) {
- throw new IllegalArgumentException("The specified path in node was not absolute: " + pathInNode);
- }
-
- return pathResolver.getApplicationStoragePathForNodeAdmin()
- .resolve(containerName.asString())
- .resolve(PathResolver.ROOT.relativize(pathInNode));
- }
-
- /**
- * Translates an absolute path in node agent container to an absolute path in host.
- * @param containerName name of the node agent container
- * @param pathInNode absolute path in that container
- * @return the absolute path in host pointing at the same inode
- */
- public Path pathInHostFromPathInNode(ContainerName containerName, Path pathInNode) {
- if (! pathInNode.isAbsolute()) {
- throw new IllegalArgumentException("The specified path in node was not absolute: " + pathInNode);
- }
-
- return pathResolver.getApplicationStoragePathForHost()
- .resolve(containerName.asString())
- .resolve(PathResolver.ROOT.relativize(pathInNode));
- }
-
- public Path pathInNodeUnderVespaHome(String relativePath) {
- return pathResolver.getVespaHomePathForContainer()
- .resolve(relativePath);
- }
-
- public List<String> getLogstashNodes() {
- return logstashNodes;
- }
-
- public NodeType getNodeType() { return nodeType; }
-
- public ContainerEnvironmentResolver getContainerEnvironmentResolver() {
- return containerEnvironmentResolver;
- }
-
- public Path getTrustStorePath() {
- return trustStorePath;
- }
-
- public AthenzService getConfigserverAthenzIdentity() {
- return configServerInfo.getConfigServerIdentity();
- }
-
- public AthenzService getNodeAthenzIdentity() {
- return nodeAthenzIdentity;
- }
-
- public String getCertificateDnsSuffix() {
- return certificateDnsSuffix;
- }
-
- public URI getZtsUri() {
- return ztsUri;
- }
-
- public URI getConfigserverLoadBalancerEndpoint() {
- return configServerInfo.getLoadBalancerEndpoint();
- }
-
- public boolean isNodeAgentCertEnabled() {
- return nodeAgentCertEnabled;
- }
-
- public DockerNetworking getDockerNetworking() {
- return dockerNetworking;
- }
-
- public static class Builder {
- private ConfigServerInfo configServerInfo;
- private String environment;
- private String region;
- private String system;
- private String cloud;
- private String parentHostHostname;
- private IPAddresses ipAddresses;
- private PathResolver pathResolver;
- private List<String> logstashNodes = Collections.emptyList();
- private NodeType nodeType = NodeType.tenant;
- private ContainerEnvironmentResolver containerEnvironmentResolver;
- private String certificateDnsSuffix;
- private URI ztsUri;
- private AthenzService nodeAthenzIdentity;
- private boolean nodeAgentCertEnabled;
- private Path trustStorePath;
- private DockerNetworking dockerNetworking;
-
- public Builder configServerConfig(ConfigServerConfig configServerConfig) {
- this.configServerInfo = new ConfigServerInfo(configServerConfig);
- return this;
- }
-
- public Builder configServerInfo(ConfigServerInfo configServerInfo) {
- this.configServerInfo = configServerInfo;
- return this;
- }
-
- public Builder environment(String environment) {
- this.environment = environment;
- return this;
- }
-
- public Builder region(String region) {
- this.region = region;
- return this;
- }
-
- public Builder system(String system) {
- this.system = system;
- return this;
- }
-
- public Builder cloud(String cloud) {
- this.cloud = cloud;
- return this;
- }
-
- public Builder parentHostHostname(String parentHostHostname) {
- this.parentHostHostname = parentHostHostname;
- return this;
- }
-
- public Builder ipAddresses(IPAddresses ipAddresses) {
- this.ipAddresses = ipAddresses;
- return this;
- }
-
- public Builder pathResolver(PathResolver pathResolver) {
- this.pathResolver = pathResolver;
- return this;
- }
-
- public Builder containerEnvironmentResolver(ContainerEnvironmentResolver containerEnvironmentResolver) {
- this.containerEnvironmentResolver = containerEnvironmentResolver;
- return this;
- }
-
- public Builder logstashNodes(List<String> hosts) {
- this.logstashNodes = hosts;
- return this;
- }
-
- public Builder nodeType(NodeType nodeType) {
- this.nodeType = nodeType;
- return this;
- }
-
- public Builder certificateDnsSuffix(String certificateDnsSuffix) {
- this.certificateDnsSuffix = certificateDnsSuffix;
- return this;
- }
-
- public Builder ztsUri(URI ztsUri) {
- this.ztsUri = ztsUri;
- return this;
- }
-
- public Builder nodeAthenzIdentity(AthenzService nodeAthenzIdentity) {
- this.nodeAthenzIdentity = nodeAthenzIdentity;
- return this;
- }
-
- public Builder enableNodeAgentCert(boolean nodeAgentCertEnabled) {
- this.nodeAgentCertEnabled = nodeAgentCertEnabled;
- return this;
- }
-
- public Builder trustStorePath(Path trustStorePath) {
- this.trustStorePath = trustStorePath;
- return this;
- }
-
- public Builder dockerNetworking(DockerNetworking dockerNetworking) {
- this.dockerNetworking = dockerNetworking;
- return this;
- }
-
- public Environment build() {
- return new Environment(configServerInfo,
- trustStorePath,
- environment,
- region,
- system,
- cloud,
- parentHostHostname,
- Optional.ofNullable(ipAddresses).orElseGet(IPAddressesImpl::new),
- Optional.ofNullable(pathResolver).orElseGet(PathResolver::new),
- logstashNodes,
- nodeType,
- Optional.ofNullable(containerEnvironmentResolver).orElseGet(() -> node -> ""),
- certificateDnsSuffix,
- ztsUri,
- nodeAthenzIdentity,
- nodeAgentCertEnabled,
- dockerNetworking);
- }
- }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java
deleted file mode 100644
index 433fae0e551..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.component;
-
-import com.yahoo.vespa.defaults.Defaults;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * @author freva
- */
-public class PathResolver {
- public static final Path ROOT = Paths.get("/");
- public static final Path DEFAULT_HOST_ROOT = Paths.get("/host");
- public static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage");
-
- private final Path hostRoot;
- private final Path vespaHomePathForContainer;
-
- private final Path applicationStoragePathForNodeAdmin;
- private final Path applicationStoragePathForHost;
-
- /**
- * @param hostRoot the absolute path to the root of the host's file system
- * @param vespaHomeForContainer the absolute path of Vespa home in the mount namespace of any
- * and all Docker containers managed by Node Admin.
- */
- public PathResolver(Path hostRoot, Path vespaHomeForContainer) {
- if (!hostRoot.isAbsolute()) {
- throw new IllegalArgumentException("Path to root of host file system is not absolute: " +
- hostRoot);
- }
- this.hostRoot = hostRoot;
-
- if (!vespaHomeForContainer.isAbsolute()) {
- throw new IllegalArgumentException("Path to Vespa home is not absolute: " + vespaHomeForContainer);
- }
- this.vespaHomePathForContainer = vespaHomeForContainer;
-
- this.applicationStoragePathForNodeAdmin = hostRoot.resolve(RELATIVE_APPLICATION_STORAGE_PATH);
- this.applicationStoragePathForHost = ROOT.resolve(RELATIVE_APPLICATION_STORAGE_PATH);
- }
-
- public PathResolver() {
- this(DEFAULT_HOST_ROOT, Paths.get(Defaults.getDefaults().vespaHome()));
- }
-
- /** For testing */
- public PathResolver(Path vespaHomePathForContainer, Path applicationStoragePathForNodeAdmin, Path applicationStoragePathForHost) {
- this.hostRoot = DEFAULT_HOST_ROOT;
- this.vespaHomePathForContainer = vespaHomePathForContainer;
- this.applicationStoragePathForNodeAdmin = applicationStoragePathForNodeAdmin;
- this.applicationStoragePathForHost = applicationStoragePathForHost;
- }
-
- /**
- * Returns the absolute path of the Vespa home directory in any Docker container mount namespace.
- *
- * It's a limitation of current implementation that all containers MUST have the same Vespa
- * home directory path.
- */
- public Path getVespaHomePathForContainer() {
- return vespaHomePathForContainer;
- }
-
- /** Returns the absolute path to the container storage directory for the node admin (this process). */
- public Path getApplicationStoragePathForNodeAdmin() {
- return applicationStoragePathForNodeAdmin;
- }
-
- /** Returns the absolute path to the container storage directory for the host. */
- public Path getApplicationStoragePathForHost() {
- return applicationStoragePathForHost;
- }
-
- /**
- * Returns the absolute path to the directory which is the root directory of the host
- * file system.
- */
- public Path getPathToRootOfHost() {
- return hostRoot;
- }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java
deleted file mode 100644
index 15cfa99c749..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage
-package com.yahoo.vespa.hosted.node.admin.config;
-
-import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index 7a83f00e297..41fe67f1996 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -304,9 +304,9 @@ public class DockerOperationsImpl implements DockerOperations {
context.pathInNodeUnderVespaHome("tmp"),
context.pathInNodeUnderVespaHome("var/container-data")));
- if (context.nodeType() == NodeType.proxyhost)
+ if (context.nodeType() == NodeType.proxy)
paths.add(context.pathInNodeUnderVespaHome("var/vespa-hosted/routing"));
- if (context.nodeType() == NodeType.host)
+ if (context.nodeType() == NodeType.tenant)
paths.add(varLibSia);
paths.forEach(path -> command.withVolume(context.pathOnHostFromPathInNode(path), path));
@@ -316,18 +316,18 @@ public class DockerOperationsImpl implements DockerOperations {
if (isInfrastructureHost(context.nodeType()))
command.withSharedVolume(varLibSia, varLibSia);
- if (context.nodeType() == NodeType.proxyhost || context.nodeType() == NodeType.controllerhost)
+ if (context.nodeType() == NodeType.proxy || context.nodeType() == NodeType.controller)
command.withSharedVolume(Paths.get("/opt/yahoo/share/ssl/certs"), Paths.get("/opt/yahoo/share/ssl/certs"));
- if (context.nodeType() == NodeType.host)
+ if (context.nodeType() == NodeType.tenant)
command.withSharedVolume(Paths.get("/var/zpe"), context.pathInNodeUnderVespaHome("var/zpe"));
}
/** Returns whether given nodeType is a Docker host for infrastructure nodes */
private static boolean isInfrastructureHost(NodeType nodeType) {
- return nodeType == NodeType.confighost ||
- nodeType == NodeType.proxyhost ||
- nodeType == NodeType.controllerhost;
+ return nodeType == NodeType.config ||
+ nodeType == NodeType.proxy ||
+ nodeType == NodeType.controller;
}
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java
deleted file mode 100644
index ce751548f75..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java
+++ /dev/null
@@ -1,223 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.logging;
-
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.component.Environment;
-
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-/**
- * @author mortent
- */
-public class FilebeatConfigProvider {
-
- private static final String TENANT_FIELD = "%%TENANT%%";
- private static final String APPLICATION_FIELD = "%%APPLICATION%%";
- private static final String INSTANCE_FIELD = "%%INSTANCE%%";
- private static final String ENVIRONMENT_FIELD = "%%ENVIRONMENT%%";
- private static final String REGION_FIELD = "%%REGION%%";
- private static final String FILEBEAT_SPOOL_SIZE_FIELD = "%%FILEBEAT_SPOOL_SIZE%%";
- private static final String LOGSTASH_HOSTS_FIELD = "%%LOGSTASH_HOSTS%%";
- private static final String LOGSTASH_WORKERS_FIELD = "%%LOGSTASH_WORKERS%%";
- private static final String LOGSTASH_BULK_MAX_SIZE_FIELD = "%%LOGSTASH_BULK_MAX_SIZE%%";
-
- private static final int logstashWorkers = 3;
- private static final int logstashBulkMaxSize = 2048;
- private final Environment environment;
-
- public FilebeatConfigProvider(Environment environment) {
- this.environment = environment;
- }
-
- public Optional<String> getConfig(NodeAgentContext context, NodeSpec node) {
-
- if (environment.getLogstashNodes().size() == 0 || !node.getOwner().isPresent()) {
- return Optional.empty();
- }
- NodeSpec.Owner owner = node.getOwner().get();
- int spoolSize = environment.getLogstashNodes().size() * logstashWorkers * logstashBulkMaxSize;
- String logstashNodeString = environment.getLogstashNodes().stream()
- .map(this::addQuotes)
- .collect(Collectors.joining(","));
- return Optional.of(getTemplate(context)
- .replaceAll(ENVIRONMENT_FIELD, environment.getEnvironment())
- .replaceAll(REGION_FIELD, environment.getRegion())
- .replaceAll(FILEBEAT_SPOOL_SIZE_FIELD, Integer.toString(spoolSize))
- .replaceAll(LOGSTASH_HOSTS_FIELD, logstashNodeString)
- .replaceAll(LOGSTASH_WORKERS_FIELD, Integer.toString(logstashWorkers))
- .replaceAll(LOGSTASH_BULK_MAX_SIZE_FIELD, Integer.toString(logstashBulkMaxSize))
- .replaceAll(TENANT_FIELD, owner.getTenant())
- .replaceAll(APPLICATION_FIELD, owner.getApplication())
- .replaceAll(INSTANCE_FIELD, owner.getInstance()));
- }
-
- private String addQuotes(String logstashNode) {
- return logstashNode.startsWith("\"")
- ? logstashNode
- : String.format("\"%s\"", logstashNode);
- }
-
- private String getTemplate(NodeAgentContext context) {
- return "################### Filebeat Configuration Example #########################\n" +
- "\n" +
- "############################# Filebeat ######################################\n" +
- "filebeat:\n" +
- " # List of prospectors to fetch data.\n" +
- " prospectors:\n" +
- "\n" +
- " # vespa\n" +
- " - paths:\n" +
- " - " + context.pathInNodeUnderVespaHome("logs/vespa/vespa.log") + "\n" +
- " exclude_files: [\".gz$\"]\n" +
- " document_type: vespa\n" +
- " fields:\n" +
- " HV-tenant: %%TENANT%%\n" +
- " HV-application: %%APPLICATION%%\n" +
- " HV-instance: %%INSTANCE%%\n" +
- " HV-region: %%REGION%%\n" +
- " HV-environment: %%ENVIRONMENT%%\n" +
- " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" +
- " fields_under_root: true\n" +
- " close_older: 20m\n" +
- " force_close_files: true\n" +
- "\n" +
- " # vespa qrs\n" +
- " - paths:\n" +
- " - " + context.pathInNodeUnderVespaHome("logs/vespa/qrs/QueryAccessLog.*.*") + "\n" +
- " exclude_files: [\".gz$\"]\n" +
- " exclude_lines: [\"reserved-for-internal-use/feedapi\"]\n" +
- " document_type: vespa-qrs\n" +
- " fields:\n" +
- " HV-tenant: %%TENANT%%\n" +
- " HV-application: %%APPLICATION%%\n" +
- " HV-instance: %%INSTANCE%%\n" +
- " HV-region: %%REGION%%\n" +
- " HV-environment: %%ENVIRONMENT%%\n" +
- " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" +
- " fields_under_root: true\n" +
- " close_older: 20m\n" +
- " force_close_files: true\n" +
- "\n" +
- " # General filebeat configuration options\n" +
- " #\n" +
- " # Event count spool threshold - forces network flush if exceeded\n" +
- " spool_size: %%FILEBEAT_SPOOL_SIZE%%\n" +
- "\n" +
- " # Defines how often the spooler is flushed. After idle_timeout the spooler is\n" +
- " # Flush even though spool_size is not reached.\n" +
- " #idle_timeout: 5s\n" +
- " publish_async: false\n" +
- "\n" +
- " # Name of the registry file. Per default it is put in the current working\n" +
- " # directory. In case the working directory is changed after when running\n" +
- " # filebeat again, indexing starts from the beginning again.\n" +
- " registry_file: /var/lib/filebeat/registry\n" +
- "\n" +
- " # Full Path to directory with additional prospector configuration files. Each file must end with .yml\n" +
- " # These config files must have the full filebeat config part inside, but only\n" +
- " # the prospector part is processed. All global options like spool_size are ignored.\n" +
- " # The config_dir MUST point to a different directory then where the main filebeat config file is in.\n" +
- " #config_dir:\n" +
- "\n" +
- "###############################################################################\n" +
- "############################# Libbeat Config ##################################\n" +
- "# Base config file used by all other beats for using libbeat features\n" +
- "\n" +
- "############################# Output ##########################################\n" +
- "\n" +
- "# Configure what outputs to use when sending the data collected by the beat.\n" +
- "# Multiple outputs may be used.\n" +
- "output:\n" +
- "\n" +
- " ### Logstash as output\n" +
- " logstash:\n" +
- " # The Logstash hosts\n" +
- " hosts: [%%LOGSTASH_HOSTS%%]\n" +
- "\n" +
- " timeout: 15\n" +
- "\n" +
- " # Number of workers per Logstash host.\n" +
- " worker: %%LOGSTASH_WORKERS%%\n" +
- "\n" +
- " # Set gzip compression level.\n" +
- " compression_level: 3\n" +
- "\n" +
- " # Optional load balance the events between the Logstash hosts\n" +
- " loadbalance: true\n" +
- "\n" +
- " # Optional index name. The default index name depends on the each beat.\n" +
- " # For Packetbeat, the default is set to packetbeat, for Topbeat\n" +
- " # top topbeat and for Filebeat to filebeat.\n" +
- " #index: filebeat\n" +
- "\n" +
- " bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%%\n" +
- "\n" +
- " # Optional TLS. By default is off.\n" +
- " #tls:\n" +
- " # List of root certificates for HTTPS server verifications\n" +
- " #certificate_authorities: [\"/etc/pki/root/ca.pem\"]\n" +
- "\n" +
- " # Certificate for TLS client authentication\n" +
- " #certificate: \"/etc/pki/client/cert.pem\"\n" +
- "\n" +
- " # Client Certificate Key\n" +
- " #certificate_key: \"/etc/pki/client/cert.key\"\n" +
- "\n" +
- " # Controls whether the client verifies server certificates and host name.\n" +
- " # If insecure is set to true, all server host names and certificates will be\n" +
- " # accepted. In this mode TLS based connections are susceptible to\n" +
- " # man-in-the-middle attacks. Use only for testing.\n" +
- " #insecure: true\n" +
- "\n" +
- " # Configure cipher suites to be used for TLS connections\n" +
- " #cipher_suites: []\n" +
- "\n" +
- " # Configure curve types for ECDHE based cipher suites\n" +
- " #curve_types: []\n" +
- "\n" +
- "############################# Shipper #########################################\n" +
- "\n" +
- "shipper:\n" +
- "\n" +
- "############################# Logging #########################################\n" +
- "\n" +
- "# There are three options for the log ouput: syslog, file, stderr.\n" +
- "# Under Windos systems, the log files are per default sent to the file output,\n" +
- "# under all other system per default to syslog.\n" +
- "logging:\n" +
- "\n" +
- " # Send all logging output to syslog. On Windows default is false, otherwise\n" +
- " # default is true.\n" +
- " to_syslog: false\n" +
- "\n" +
- " # Write all logging output to files. Beats automatically rotate files if rotateeverybytes\n" +
- " # limit is reached.\n" +
- " to_files: true\n" +
- "\n" +
- " # To enable logging to files, to_files option has to be set to true\n" +
- " files:\n" +
- " # The directory where the log files will written to.\n" +
- " path: " + context.pathInNodeUnderVespaHome("logs/filebeat") + "\n" +
- "\n" +
- " # The name of the files where the logs are written to.\n" +
- " name: filebeat\n" +
- "\n" +
- " # Configure log file size limit. If limit is reached, log file will be\n" +
- " # automatically rotated\n" +
- " rotateeverybytes: 10485760 # = 10MB\n" +
- "\n" +
- " # Number of rotated log files to keep. Oldest files will be deleted first.\n" +
- " keepfiles: 7\n" +
- "\n" +
- " # Enable debug output for selected components. To enable all selectors use [\"*\"]\n" +
- " # Other available selectors are beat, publish, service\n" +
- " # Multiple selectors can be chained.\n" +
- " #selectors: [ ]\n" +
- "\n" +
- " # Sets log level. The default log level is error.\n" +
- " # Available log levels are: critical, error, warning, info, debug\n" +
- " level: warning\n";
- }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java
deleted file mode 100644
index cf010121c2a..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java
+++ /dev/null
@@ -1,177 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.maintenance;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.Path;
-import java.nio.file.attribute.FileTime;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Optional;
-import java.util.logging.Logger;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * @author freva
- */
-public class FileHelper {
- private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName());
-
- /**
- * (Recursively) deletes files if they match all the criteria, also deletes empty directories.
- *
- * @param basePath Base path from where to start the search
- * @param maxAge Delete files older (last modified date) than maxAge
- * @param fileNameRegex Delete files where filename matches fileNameRegex
- * @param recursive Delete files in sub-directories (with the same criteria)
- */
- public static void deleteFiles(Path basePath, Duration maxAge, Optional<String> fileNameRegex, boolean recursive) throws IOException {
- Pattern fileNamePattern = fileNameRegex.map(Pattern::compile).orElse(null);
-
- for (Path path : listContentsOfDirectory(basePath)) {
- if (Files.isDirectory(path)) {
- if (recursive) {
- deleteFiles(path, maxAge, fileNameRegex, true);
- if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) {
- logger.warning("Could not delete directory: " + path.toAbsolutePath());
- }
- }
- } else if (isPatternMatchingFilename(fileNamePattern, path) &&
- isTimeSinceLastModifiedMoreThan(path, maxAge)) {
- if (! Files.deleteIfExists(path)) {
- logger.warning("Could not delete file: " + path.toAbsolutePath());
- }
- }
- }
- }
-
- /**
- * Deletes all files in target directory except the n most recent (by modified date)
- *
- * @param basePath Base path to delete from
- * @param nMostRecentToKeep Number of most recent files to keep
- */
- static void deleteFilesExceptNMostRecent(Path basePath, int nMostRecentToKeep) throws IOException {
- if (nMostRecentToKeep < 1) {
- throw new IllegalArgumentException("Number of files to keep must be a positive number");
- }
-
- List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream()
- .filter(Files::isRegularFile)
- .sorted(Comparator.comparing(FileHelper::getLastModifiedTime))
- .skip(nMostRecentToKeep)
- .collect(Collectors.toList());
-
- for (Path path : pathsInDeleteDir) {
- if (!Files.deleteIfExists(path)) {
- logger.warning("Could not delete file: " + path.toAbsolutePath());
- }
- }
- }
-
- static void deleteFilesLargerThan(Path basePath, long sizeInBytes) throws IOException {
- for (Path path : listContentsOfDirectory(basePath)) {
- if (Files.isDirectory(path)) {
- deleteFilesLargerThan(path, sizeInBytes);
- } else {
- if (Files.size(path) > sizeInBytes && !Files.deleteIfExists(path)) {
- logger.warning("Could not delete file: " + path.toAbsolutePath());
- }
- }
- }
- }
-
- /**
- * Deletes directories and their contents if they match all the criteria
- *
- * @param basePath Base path to delete the directories from
- * @param maxAge Delete directories older (last modified date) than maxAge
- * @param dirNameRegex Delete directories where directory name matches dirNameRegex
- */
- public static void deleteDirectories(Path basePath, Duration maxAge, Optional<String> dirNameRegex) throws IOException {
- Pattern dirNamePattern = dirNameRegex.map(Pattern::compile).orElse(null);
-
- for (Path path : listContentsOfDirectory(basePath)) {
- if (Files.isDirectory(path) && isPatternMatchingFilename(dirNamePattern, path)) {
- boolean mostRecentFileModifiedBeforeMaxAge = getMostRecentlyModifiedFileIn(path)
- .map(mostRecentlyModified -> isTimeSinceLastModifiedMoreThan(mostRecentlyModified, maxAge))
- .orElse(true);
-
- if (mostRecentFileModifiedBeforeMaxAge) {
- deleteFiles(path, Duration.ZERO, Optional.empty(), true);
- if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) {
- logger.warning("Could not delete directory: " + path.toAbsolutePath());
- }
- }
- }
- }
- }
-
- /**
- * Similar to rm -rf file:
- * - It's not an error if file doesn't exist
- * - If file is a directory, it and all content is removed
- * - For symlinks: Only the symlink is removed, not what the symlink points to
- */
- public static void recursiveDelete(Path basePath) throws IOException {
- if (Files.isDirectory(basePath)) {
- for (Path path : listContentsOfDirectory(basePath)) {
- recursiveDelete(path);
- }
- }
-
- Files.deleteIfExists(basePath);
- }
-
- public static void moveIfExists(Path from, Path to) throws IOException {
- if (Files.exists(from)) {
- Files.move(from, to);
- }
- }
-
- private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException {
- return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime));
- }
-
- private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) {
- Instant nowMinusDuration = Instant.now().minus(duration);
- Instant lastModified = getLastModifiedTime(path).toInstant();
-
- // Return true also if they are equal for test stability
- // (lastModified <= nowMinusDuration) is the same as !(lastModified > nowMinusDuration)
- return !lastModified.isAfter(nowMinusDuration);
- }
-
- private static boolean isPatternMatchingFilename(Pattern pattern, Path path) {
- return pattern == null || pattern.matcher(path.getFileName().toString()).find();
- }
-
- /**
- * @return list all files in a directory, returns empty list if directory does not exist
- */
- public static List<Path> listContentsOfDirectory(Path basePath) {
- try (Stream<Path> directoryStream = Files.list(basePath)) {
- return directoryStream.collect(Collectors.toList());
- } catch (NoSuchFileException ignored) {
- return Collections.emptyList();
- } catch (IOException e) {
- throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e);
- }
- }
-
- static FileTime getLastModifiedTime(Path path) {
- try {
- return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS);
- } catch (IOException e) {
- throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e);
- }
- }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 6a9f0a69249..c9e4a8fac7d 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -111,8 +111,8 @@ public class StorageMaintainer {
//routing-configage
Path routingAgeCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60,
- routingAgeCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(),
- "-m", "90", "-a", "routing-configage");
+ routingAgeCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf.tmp").toString(),
+ "-m", "1", "-a", "routing-configage", "--ignore_file_not_found");
configs.add(annotatedCheck(context, node, routingAgeSchedule));
//ssl-check
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
index bbea505b19d..22093258930 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
@@ -70,7 +70,7 @@ public class AthenzCredentialsMaintainer {
private final CsrGenerator csrGenerator;
// Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts
- private Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>();
+ private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>();
public AthenzCredentialsMaintainer(URI ztsEndpoint,
Path trustStorePath,
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 395b4d458a2..ef137c55ffb 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -337,7 +337,7 @@ public class NodeAgentImpl implements NodeAgent {
}
if (node.getWantedDockerImage().isPresent() && !node.getWantedDockerImage().get().equals(existingContainer.image)) {
return Optional.of("The node is supposed to run a new Docker image: "
- + existingContainer + " -> " + node.getWantedDockerImage().get());
+ + existingContainer.image.asString() + " -> " + node.getWantedDockerImage().get().asString());
}
if (!existingContainer.state.isRunning()) {
return Optional.of("Container no longer running");
diff --git a/node-admin/src/main/resources/configdefinitions/config-server.def b/node-admin/src/main/resources/configdefinitions/config-server.def
deleted file mode 100644
index 6a088829bad..00000000000
--- a/node-admin/src/main/resources/configdefinitions/config-server.def
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-namespace=vespa.hosted.node.admin.config
-
-hosts[] string
-port int default=8080 range=[1,65535]
-scheme string default="http"
-loadBalancerHost string default=""
-configserverAthenzIdentity string default="vespa.configserver" \ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java
deleted file mode 100644
index 281c7df9a1f..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-package com.yahoo.vespa.hosted.node.admin.component;
-
-import org.junit.Test;
-
-import java.nio.file.Paths;
-
-import static org.junit.Assert.assertEquals;
-
-public class PathResolverTest {
- @Test
- public void testNodeAdminOnHost() {
- PathResolver pathResolver = new PathResolver(Paths.get("/"), Paths.get("/home/y"));
- assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForHost());
- assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForNodeAdmin());
- assertEquals(Paths.get("/"), pathResolver.getPathToRootOfHost());
- assertEquals(Paths.get("/home/y"), pathResolver.getVespaHomePathForContainer());
- }
-
- @Test
- public void testNodeAdminInContainer() {
- PathResolver pathResolver = new PathResolver(Paths.get("/host"), Paths.get("/home/y"));
- assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForHost());
- assertEquals(Paths.get("/host/home/docker/container-storage"), pathResolver.getApplicationStoragePathForNodeAdmin());
- assertEquals(Paths.get("/host"), pathResolver.getPathToRootOfHost());
- assertEquals(Paths.get("/home/y"), pathResolver.getVespaHomePathForContainer());
- }
-} \ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java
deleted file mode 100644
index d4fadabe695..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java
+++ /dev/null
@@ -1,129 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.logging;
-
-import com.google.common.collect.ImmutableList;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.component.Environment;
-import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
-import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
-import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
-import com.yahoo.vespa.hosted.provision.Node;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.core.IsNot.not;
-import static org.junit.Assert.*;
-
-/**
- * @author mortent
- */
-public class FilebeatConfigProviderTest {
-
- private static final String tenant = "vespa";
- private static final String application = "music";
- private static final String instance = "default";
- private static final String environment = "prod";
- private static final String region = "us-north-1";
- private static final String system = "main";
- private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2");
- private final NodeAgentContext context = new NodeAgentContextImpl.Builder("node-123.hostname.tld").build();
-
- @Test
- public void it_replaces_all_fields_correctly() {
- FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes));
-
- Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance));
-
- assertTrue(config.isPresent());
- String configString = config.get();
- assertThat(configString, not(containsString("%%")));
- }
-
- @Test
- public void it_does_not_generate_config_when_no_logstash_nodes() {
- Environment env = getEnvironment(Collections.emptyList());
-
- FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(env);
- Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance));
- assertFalse(config.isPresent());
- }
-
- @Test
- public void it_does_not_generate_config_for_nodes_wihout_owner() {
- FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes));
- NodeSpec node = new NodeSpec.Builder()
- .flavor("flavor")
- .state(Node.State.active)
- .nodeType(NodeType.tenant)
- .hostname("hostname")
- .minCpuCores(1)
- .minMainMemoryAvailableGb(1)
- .minDiskAvailableGb(1)
- .build();
- Optional<String> config = filebeatConfigProvider.getConfig(context, node);
- assertFalse(config.isPresent());
- }
-
- @Test
- public void it_generates_correct_index_source() {
- assertThat(getConfigString(), containsString("index_source: \"hosted-instance_vespa_music_us-north-1_prod_default\""));
- }
-
- @Test
- public void it_sets_logstash_nodes_properly() {
- assertThat(getConfigString(), containsString("hosts: [\"logstash1\",\"logstash2\"]"));
- }
-
- @Test
- public void it_does_not_add_double_quotes() {
- Environment environment = getEnvironment(ImmutableList.of("unquoted", "\"quoted\""));
- FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment);
- Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance));
- assertThat(config.get(), containsString("hosts: [\"unquoted\",\"quoted\"]"));
- }
-
- @Test
- public void it_generates_correct_spool_size() {
- // 2 nodes, 3 workers, 2048 buffer size -> 12288
- assertThat(getConfigString(), containsString("spool_size: 12288"));
- }
-
- private String getConfigString() {
- FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes));
- NodeSpec node = createNodeRepositoryNode(tenant, application, instance);
- return filebeatConfigProvider.getConfig(context, node).orElseThrow(() -> new RuntimeException("Failed to get filebeat config"));
- }
-
- private Environment getEnvironment(List<String> logstashNodes) {
- return new Environment.Builder()
- .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder()))
- .environment(environment)
- .region(region)
- .system(system)
- .logstashNodes(logstashNodes)
- .cloud("mycloud")
- .dockerNetworking(DockerNetworking.HOST_NETWORK)
- .build();
- }
-
- private NodeSpec createNodeRepositoryNode(String tenant, String application, String instance) {
- NodeSpec.Owner owner = new NodeSpec.Owner(tenant, application, instance);
- return new NodeSpec.Builder()
- .owner(owner)
- .flavor("flavor")
- .state(Node.State.active)
- .nodeType(NodeType.tenant)
- .hostname("hostname")
- .minCpuCores(1)
- .minMainMemoryAvailableGb(1)
- .minDiskAvailableGb(1)
- .build();
- }
-
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java
deleted file mode 100644
index 6b53bc217c4..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java
+++ /dev/null
@@ -1,324 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.maintenance;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Optional;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author freva
- */
-public class FileHelperTest {
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Before
- public void initFiles() throws IOException {
- for (int i=0; i<10; i++) {
- File temp = folder.newFile("test_" + i + ".json");
- temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis());
- }
-
- for (int i=0; i<7; i++) {
- File temp = folder.newFile("test_" + i + "_file.test");
- temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis());
- }
-
- for (int i=0; i<5; i++) {
- File temp = folder.newFile(i + "-abc" + ".json");
- temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis());
- }
-
- File temp = folder.newFile("week_old_file.json");
- temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis());
- }
-
- @Test
- public void testDeleteAll() throws IOException {
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.empty(), false);
-
- assertEquals(0, getContentsOfDirectory(folder.getRoot()).length);
- }
-
- @Test
- public void testDeletePrefix() throws IOException {
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false);
-
- assertEquals(6, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file
- }
-
- @Test
- public void testDeleteSuffix() throws IOException {
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".json$"), false);
-
- assertEquals(7, getContentsOfDirectory(folder.getRoot()).length);
- }
-
- @Test
- public void testDeletePrefixAndSuffix() throws IOException {
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false);
-
- assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 7 test_*_file.test files + week_old_file
- }
-
- @Test
- public void testDeleteOld() throws IOException {
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(600), Optional.empty(), false);
-
- assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file
- }
-
- @Test
- public void testDeleteWithAllParameters() throws IOException {
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(200), Optional.of("^test_.*\\.json$"), false);
-
- assertEquals(15, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 8 (from test_*_.json)
- }
-
- @Test
- public void testDeleteWithSubDirectoriesNoRecursive() throws IOException {
- initSubDirectories();
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false);
-
- // 6 test_*.json from test_folder1/
- // + 9 test_*.json and 4 abc_*.json from test_folder2/
- // + 13 test_*.json from test_folder2/subSubFolder2/
- // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root
- // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves
- assertEquals(48, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
- }
-
- @Test
- public void testDeleteWithSubDirectoriesRecursive() throws IOException {
- initSubDirectories();
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), true);
-
- // 4 abc_*.json from test_folder2/
- // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root
- // + test_folder2/ itself
- assertEquals(18, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
- }
-
- @Test
- public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException {
- initSubDirectories();
-
- FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false);
-
- assertEquals(8, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file + 2 directories
- }
-
- @Test
- public void testGetContentsOfNonExistingDirectory() {
- Path fakePath = Paths.get("/some/made/up/dir/");
- assertEquals(Collections.emptyList(), FileHelper.listContentsOfDirectory(fakePath));
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testDeleteFilesExceptNMostRecentWithNegativeN() throws IOException {
- FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), -5);
- }
-
- @Test
- public void testDeleteFilesExceptFiveMostRecent() throws IOException {
- FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 5);
-
- assertEquals(5, getContentsOfDirectory(folder.getRoot()).length);
-
- String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"};
- String[] remainingFiles = Arrays.stream(getContentsOfDirectory(folder.getRoot()))
- .map(File::getName)
- .sorted()
- .toArray(String[]::new);
-
- assertArrayEquals(oldestFiles, remainingFiles);
- }
-
- @Test
- public void testDeleteFilesExceptNMostRecentWithLargeN() throws IOException {
- String[] filesPreDelete = folder.getRoot().list();
-
- FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 50);
-
- assertArrayEquals(filesPreDelete, folder.getRoot().list());
- }
-
- @Test
- public void testDeleteFilesLargerThan10B() throws IOException {
- initSubDirectories();
-
- File temp1 = new File(folder.getRoot(), "small_file");
- writeNBytesToFile(temp1, 50);
-
- File temp2 = new File(folder.getRoot(), "some_file");
- writeNBytesToFile(temp2, 20);
-
- File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file");
- writeNBytesToFile(temp3, 75);
-
- FileHelper.deleteFilesLargerThan(folder.getRoot().toPath(), 10);
-
- assertEquals(58, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
- assertFalse(temp1.exists() || temp2.exists() || temp3.exists());
- }
-
- @Test
- public void testDeleteDirectories() throws IOException {
- initSubDirectories();
-
- FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".*folder2"));
-
- //23 files in root
- // + 6 in test_folder1 + test_folder1 itself
- assertEquals(30, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
- }
-
- @Test
- public void testDeleteDirectoriesBasedOnAge() throws IOException {
- initSubDirectories();
- // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is
- // also older than maxAge inside the sub directory, create some files which are newer than maxAge.
- // deleteDirectories() should NOT delete folder3
- File subFolder3 = folder.newFolder("test_folder3");
- File subSubFolder3 = folder.newFolder("test_folder3", "subSubFolder3");
-
- for (int j=0; j<11; j++) {
- File.createTempFile("test_", ".json", subSubFolder3);
- }
-
- subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis());
- subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis());
-
- FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*"));
-
- //23 files in root
- // + 13 in test_folder2
- // + 13 in subSubFolder2
- // + 11 in subSubFolder3
- // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself
- assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot()));
- }
-
- @Test
- public void testRecursivelyDeleteDirectory() throws IOException {
- initSubDirectories();
- FileHelper.recursiveDelete(folder.getRoot().toPath());
- assertFalse(folder.getRoot().exists());
- }
-
- @Test
- public void testRecursivelyDeleteRegularFile() throws IOException {
- File file = folder.newFile();
- assertTrue(file.exists());
- assertTrue(file.isFile());
- FileHelper.recursiveDelete(file.toPath());
- assertFalse(file.exists());
- }
-
- @Test
- public void testRecursivelyDeleteNonExistingFile() throws IOException {
- File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile();
- assertFalse(file.exists());
- FileHelper.recursiveDelete(file.toPath());
- assertFalse(file.exists());
- }
-
- @Test
- public void testInitSubDirectories() throws IOException {
- initSubDirectories();
- assertTrue(folder.getRoot().exists());
- assertTrue(folder.getRoot().isDirectory());
-
- Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1");
- assertTrue(test_folder1.toFile().exists());
- assertTrue(test_folder1.toFile().isDirectory());
-
- Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2");
- assertTrue(test_folder2.toFile().exists());
- assertTrue(test_folder2.toFile().isDirectory());
-
- Path subSubFolder2 = test_folder2.resolve("subSubFolder2");
- assertTrue(subSubFolder2.toFile().exists());
- assertTrue(subSubFolder2.toFile().isDirectory());
- }
-
- @Test
- public void testDoesNotFailOnLastModifiedOnSymLink() throws IOException {
- Path symPath = folder.getRoot().toPath().resolve("symlink");
- Path fakePath = Paths.get("/some/not/existant/file");
-
- Files.createSymbolicLink(symPath, fakePath);
- assertTrue(Files.isSymbolicLink(symPath));
- assertFalse(Files.exists(fakePath));
-
- // Not possible to set modified time on symlink in java, so just check that it doesn't crash
- FileHelper.getLastModifiedTime(symPath).toInstant();
- }
-
- private void initSubDirectories() throws IOException {
- File subFolder1 = folder.newFolder("test_folder1");
- File subFolder2 = folder.newFolder("test_folder2");
- File subSubFolder2 = folder.newFolder("test_folder2", "subSubFolder2");
-
- for (int j=0; j<6; j++) {
- File temp = File.createTempFile("test_", ".json", subFolder1);
- temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis());
- }
-
- for (int j=0; j<9; j++) {
- File.createTempFile("test_", ".json", subFolder2);
- }
-
- for (int j=0; j<4; j++) {
- File.createTempFile("abc_", ".txt", subFolder2);
- }
-
- for (int j=0; j<13; j++) {
- File temp = File.createTempFile("test_", ".json", subSubFolder2);
- temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis());
- }
-
- //Must be after all the files have been created
- subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis());
- subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis());
- subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis());
- }
-
- private static int getNumberOfFilesAndDirectoriesIn(File folder) {
- int total = 0;
- for (File file : getContentsOfDirectory(folder)) {
- if (file.isDirectory()) {
- total += getNumberOfFilesAndDirectoriesIn(file);
- }
- total++;
- }
-
- return total;
- }
-
- private static void writeNBytesToFile(File file, int nBytes) throws IOException {
- Files.write(file.toPath(), new byte[nBytes]);
- }
-
- private static File[] getContentsOfDirectory(File directory) {
- File[] directoryContents = directory.listFiles();
-
- return directoryContents == null ? new File[0] : directoryContents;
- }
-}