diff options
Diffstat (limited to 'node-admin')
31 files changed, 571 insertions, 247 deletions
diff --git a/node-admin/README_MAC.md b/node-admin/README_MAC.md index 75a67f6a29c..71f0cae0d7d 100644 --- a/node-admin/README_MAC.md +++ b/node-admin/README_MAC.md @@ -43,7 +43,7 @@ scripts/setup-route-and-hosts-osx.sh The script will prompt you to continue as this will alter your routing table and /etc/hosts file. If your local zone is up and running, the config server should respond to this: ``` -curl config-server:19071 +curl config-server:4080 ``` If you don't want your `/etc/hosts` file to be changed, the diff --git a/node-admin/include/nodectl-instance.sh b/node-admin/include/nodectl-instance.sh index 5a6665dbdc7..a8d872b314e 100755 --- a/node-admin/include/nodectl-instance.sh +++ b/node-admin/include/nodectl-instance.sh @@ -103,7 +103,7 @@ stop() { $echo $VESPA_HOME/bin/vespa-routing vip -u chef out if has_searchnode; then - $echo $VESPA_HOME/bin/vespa-proton-cmd --local triggerFlush + $echo $VESPA_HOME/bin/vespa-proton-cmd --local prepareRestart fi if has_container; then diff --git a/node-admin/scripts/app.sh b/node-admin/scripts/app.sh index 83754413508..2757d637dc8 100755 --- a/node-admin/scripts/app.sh +++ b/node-admin/scripts/app.sh @@ -101,7 +101,7 @@ function DeployApp { # Create tenant echo -n "Creating tenant... " local create_tenant_response - if create_tenant_response=$(curl --silent --show-error -X PUT "http://$CONFIG_SERVER_HOSTNAME:$CONFIG_SERVER_PORT/application/v2/tenant/$TENANT_NAME" 2>&1) + if create_tenant_response=$(curl --silent --show-error -X PUT "http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/application/v2/tenant/$TENANT_NAME" 2>&1) then if ! [[ "$create_tenant_response" =~ "Tenant $TENANT_NAME created" ]] && ! [[ "$create_tenant_response" =~ "already exists" ]] @@ -131,7 +131,7 @@ function UndeployApp { local app_name=default local output echo -n "Removing application $TENANT_NAME:$app_name... " - if ! output=$(curl --silent --show-error -X DELETE "http://$CONFIG_SERVER_HOSTNAME:$CONFIG_SERVER_PORT/application/v2/tenant/$TENANT_NAME/application/$app_name") + if ! output=$(curl --silent --show-error -X DELETE "http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/application/v2/tenant/$TENANT_NAME/application/$app_name") then echo Fail "Failed to remove application: $output" diff --git a/node-admin/scripts/common.sh b/node-admin/scripts/common.sh index d07b4adcc5a..6a10fb71a99 100644 --- a/node-admin/scripts/common.sh +++ b/node-admin/scripts/common.sh @@ -28,7 +28,7 @@ declare -r NODE_ADMIN_CONTAINER_NAME=node-admin declare -r CONFIG_SERVER_CONTAINER_NAME=config-server declare -r CONFIG_SERVER_HOSTNAME="$CONFIG_SERVER_CONTAINER_NAME" declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1" -declare -r CONFIG_SERVER_PORT=19071 +declare -r VESPA_WEB_SERVICE_PORT=4080 # E.g. config server port declare -r DEFAULT_HOSTED_VESPA_REGION=local-region declare -r DEFAULT_HOSTED_VESPA_ENVIRONMENT=prod diff --git a/node-admin/scripts/config-server.sh b/node-admin/scripts/config-server.sh index 60b05d4b3cd..f8e0f1a60e4 100755 --- a/node-admin/scripts/config-server.sh +++ b/node-admin/scripts/config-server.sh @@ -124,7 +124,7 @@ function Start { then # Wait for config server to come up echo -n "Waiting for healthy Config Server (~30s)" - local url="http://$CONFIG_SERVER_HOSTNAME:19071/state/v1/health" + local url="http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/state/v1/health" while ! curl --silent --fail --max-time 1 "$url" >/dev/null do echo -n . diff --git a/node-admin/scripts/node-repo.sh b/node-admin/scripts/node-repo.sh index 94173a6726b..2e113843916 100755 --- a/node-admin/scripts/node-repo.sh +++ b/node-admin/scripts/node-repo.sh @@ -3,6 +3,8 @@ set -e +declare -r VESPA_WEB_SERVICE_PORT=4080 + # Output from InnerCurlNodeRepo, see there for details. declare CURL_RESPONSE @@ -162,7 +164,7 @@ function ProvisionNode { local config_server_hostname="$1" local json="$2" - local url="http://$config_server_hostname:19071/nodes/v2/node" + local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/node" CurlOrFail -H "Content-Type: application/json" -X POST -d "$json" "$url" } @@ -172,7 +174,7 @@ function SetNodeState { local hostname="$2" local state="$3" - local url="http://$config_server_hostname:19071/nodes/v2/state/$state/$hostname" + local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/state/$state/$hostname" CurlOrFail -X PUT "$url" } @@ -284,7 +286,7 @@ function RemoveCommand { local hostname for hostname in "$@" do - local url="http://$config_server_hostname:19071/nodes/v2/node/$hostname" + local url="http://$config_server_hostname:$VESPA_WEB_SERVICE_PORT/nodes/v2/node/$hostname" CurlOrFail -X DELETE "$url" echo -n . done diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index a5dea070285..c746afa0e85 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -8,11 +8,13 @@ </handler> <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl" bundle="node-admin"/> <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> + <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> <config name='vespa.hosted.dockerapi.docker'> - <caCertPath>/host/docker/certs/ca_cert.pem</caCertPath> - <clientCertPath>/host/docker/certs/client_cert.pem</clientCertPath> - <clientKeyPath>/host/docker/certs/client_key.pem</clientKeyPath> + <uri>tcp://localhost:2376</uri> + <caCertPath></caCertPath> + <clientCertPath></clientCertPath> + <clientKeyPath></clientKeyPath> </config> </jdisc> </services> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index bfd46e1453e..0877275f93d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -27,8 +27,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.regex.Matcher; @@ -50,24 +52,29 @@ public class DockerOperationsImpl implements DockerOperations { private static final Pattern VESPA_VERSION_PATTERN = Pattern.compile("^(\\S*)$", Pattern.MULTILINE); - private static final List<String> DIRECTORIES_TO_MOUNT = Arrays.asList( - getDefaults().underVespaHome("logs"), - getDefaults().underVespaHome("var/cache"), - getDefaults().underVespaHome("var/crash"), - getDefaults().underVespaHome("var/db/jdisc"), - getDefaults().underVespaHome("var/db/vespa"), - getDefaults().underVespaHome("var/jdisc_container"), - getDefaults().underVespaHome("var/jdisc_core"), - getDefaults().underVespaHome("var/maven"), - getDefaults().underVespaHome("var/run"), - getDefaults().underVespaHome("var/scoreboards"), - getDefaults().underVespaHome("var/service"), - getDefaults().underVespaHome("var/share"), - getDefaults().underVespaHome("var/spool"), - getDefaults().underVespaHome("var/vespa"), - getDefaults().underVespaHome("var/yca"), - getDefaults().underVespaHome("var/ycore++"), - getDefaults().underVespaHome("var/zookeeper")); + // Map of directories to mount and whether they should be writeable by everyone + private static final Map<String, Boolean> DIRECTORIES_TO_MOUNT = new HashMap<>(); + static { + DIRECTORIES_TO_MOUNT.put("/metrics-share", true); + DIRECTORIES_TO_MOUNT.put("/etc/yamas-agent", true); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/cache"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/crash"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/db/jdisc"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/db/vespa"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/jdisc_container"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/jdisc_core"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/maven"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/run"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/scoreboards"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/service"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/share"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/spool"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/vespa"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/yca"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/ycore++"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/zookeeper"), false); + } private final Docker docker; private final Environment environment; @@ -119,34 +126,32 @@ public class DockerOperationsImpl implements DockerOperations { } private void configureContainer(ContainerNodeSpec nodeSpec) { - Path yamasAgentTempFolder = Paths.get("/tmp/yamas_schedule_" + System.currentTimeMillis() + "/yamas-agent/"); - yamasAgentTempFolder.toFile().mkdirs(); + final Path yamasAgentFolder = Paths.get("/etc/yamas-agent/"); - // Path to the executeable that the secret-agent will run to gather metrics - Path systemCheckPath = Paths.get("/usr/bin/yms_check_system"); - // Path to the secret-agent schedule file - Path systemCheckSchedulePath = yamasAgentTempFolder.resolve("system-checks.yaml"); - // Contents of the secret-agent schedule file - String systemCheckSchedule = generateSecretAgentSchedule(nodeSpec, "system-checks", 60, systemCheckPath, - "-l", "/var/secret-agent/custom/"); + Path diskUsageCheckPath = Paths.get("/bin/cat"); + Path diskUsageCheckSchedulePath = yamasAgentFolder.resolve("disk-usage.yaml"); + String diskUsageCheckSchedule = generateSecretAgentSchedule(nodeSpec, "disk-usage", 60, diskUsageCheckPath, + "/metrics-share/disk.usage"); Path vespaCheckPath = Paths.get("/home/y/libexec/yms/yms_check_vespa"); - Path vespaCheckSchedulePath = yamasAgentTempFolder.resolve("vespa.yaml"); + Path vespaCheckSchedulePath = yamasAgentFolder.resolve("vespa.yaml"); String vespaCheckSchedule = generateSecretAgentSchedule(nodeSpec, "vespa", 60, vespaCheckPath, "all"); try { - Files.write(systemCheckSchedulePath, systemCheckSchedule.getBytes()); - systemCheckSchedulePath.toFile().setReadable(true, false); // Give everyone read access to the schedule file - - Files.write(vespaCheckSchedulePath, vespaCheckSchedule.getBytes()); - vespaCheckSchedulePath.toFile().setReadable(true, false); + writeSecretAgentSchedule(nodeSpec.containerName, diskUsageCheckSchedulePath, diskUsageCheckSchedule); + writeSecretAgentSchedule(nodeSpec.containerName, vespaCheckSchedulePath, vespaCheckSchedule); } catch (IOException e) { e.printStackTrace(); } - docker.copyArchiveToContainer(yamasAgentTempFolder.toString(), nodeSpec.containerName, "/etc/"); docker.executeInContainer(nodeSpec.containerName, "service", "yamas-agent", "restart"); } + private void writeSecretAgentSchedule(ContainerName containerName, Path schedulePath, String secretAgentSchedule) throws IOException { + Path scheduleFilePath = Maintainer.pathInNodeAdminFromPathInNode(containerName, schedulePath.toString()); + Files.write(scheduleFilePath, secretAgentSchedule.getBytes()); + scheduleFilePath.toFile().setReadable(true, false); // Give everyone read access to the schedule file + } + String generateSecretAgentSchedule(ContainerNodeSpec nodeSpec, String id, int interval, Path pathToCheck, String... args) { StringBuilder stringBuilder = new StringBuilder() @@ -292,7 +297,7 @@ public class DockerOperationsImpl implements DockerOperations { .withEnvironment("CONFIG_SERVER_ADDRESS", configServers); command.withVolume("/etc/hosts", "/etc/hosts"); - for (String pathInNode : DIRECTORIES_TO_MOUNT) { + for (String pathInNode : DIRECTORIES_TO_MOUNT.keySet()) { String pathInHost = Maintainer.pathInHostFromPathInNode(nodeSpec.containerName, pathInNode).toString(); command = command.withVolume(pathInHost, pathInNode); } @@ -304,6 +309,9 @@ public class DockerOperationsImpl implements DockerOperations { long minMainMemoryAvailableMb = (long) (nodeSpec.minMainMemoryAvailableGb.get() * 1024); if (minMainMemoryAvailableMb > 0) { command.withMemoryInMb(minMainMemoryAvailableMb); + // TOTAL_MEMORY_MB is used to make any jdisc container think the machine + // only has this much physical memory (overrides total memory reported by `free -m`). + command.withEnvironment("TOTAL_MEMORY_MB", Long.toString(minMainMemoryAvailableMb)); } } @@ -317,6 +325,9 @@ public class DockerOperationsImpl implements DockerOperations { } else { docker.startContainer(nodeSpec.containerName); } + + DIRECTORIES_TO_MOUNT.entrySet().stream().filter(Map.Entry::getValue).forEach(entry -> + docker.executeInContainer(nodeSpec.containerName, "sudo", "chmod", "-R", "a+w", entry.getKey())); } catch (UnknownHostException e) { throw new RuntimeException("Failed to create container " + nodeSpec.containerName.asString(), e); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/MaintenanceScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/MaintenanceScheduler.java deleted file mode 100644 index 711edf4544d..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/MaintenanceScheduler.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import com.yahoo.vespa.hosted.dockerapi.ContainerName; - -import java.io.IOException; - -/** - * @author valerijf - */ -public interface MaintenanceScheduler { - void removeOldFilesFromNode(ContainerName containerName); - - void cleanNodeAdmin(); - - void deleteContainerStorage(ContainerName containerName) throws IOException; -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/MaintenanceSchedulerImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/MaintenanceSchedulerImpl.java deleted file mode 100644 index db6b40f4c73..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/MaintenanceSchedulerImpl.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import com.yahoo.io.IOUtils; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; -import com.yahoo.vespa.hosted.node.maintenance.DeleteOldAppData; -import com.yahoo.vespa.hosted.node.maintenance.Maintainer; - -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; - -/** - * @author valerijf - */ -public class MaintenanceSchedulerImpl implements MaintenanceScheduler { - private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(MaintenanceSchedulerImpl.class); - - private static final String[] baseArguments = {"sudo", "/home/y/libexec/vespa/node-admin/maintenance.sh"}; - - @Override - public void removeOldFilesFromNode(ContainerName containerName) { - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(MaintenanceSchedulerImpl.class, containerName); - - String[] pathsToClean = {"/home/y/logs/elasticsearch2", "/home/y/logs/logstash2", - "/home/y/logs/daemontools_y", "/home/y/logs/nginx", "/home/y/logs/vespa"}; - for (String pathToClean : pathsToClean) { - File path = Maintainer.pathInNodeAdminFromPathInNode(containerName, pathToClean).toFile(); - if (path.exists()) { - DeleteOldAppData.deleteFiles(path.getAbsolutePath(), Duration.ofDays(3).getSeconds(), ".*\\.log\\..+", false); - DeleteOldAppData.deleteFiles(path.getAbsolutePath(), Duration.ofDays(3).getSeconds(), ".*QueryAccessLog.*", false); - } - } - - File logArchiveDir = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/home/y/logs/vespa/logarchive").toFile(); - if (logArchiveDir.exists()) { - DeleteOldAppData.deleteFiles(logArchiveDir.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false); - } - - File fileDistrDir = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/home/y/var/db/vespa/filedistribution").toFile(); - if (fileDistrDir.exists()) { - DeleteOldAppData.deleteFiles(fileDistrDir.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false); - } - - execute(logger, Maintainer.JOB_CLEAN_CORE_DUMPS); - } - - @Override - public void cleanNodeAdmin() { - execute(NODE_ADMIN_LOGGER, Maintainer.JOB_DELETE_OLD_APP_DATA); - execute(NODE_ADMIN_LOGGER, Maintainer.JOB_CLEAN_HOME); - - File nodeAdminJDiskLogsPath = Maintainer.pathInNodeAdminFromPathInNode(new ContainerName("node-admin"), - "/home/y/logs/jdisc_core/").toFile(); - DeleteOldAppData.deleteFiles(nodeAdminJDiskLogsPath.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false); - } - - @Override - public void deleteContainerStorage(ContainerName containerName) throws IOException { - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(MaintenanceSchedulerImpl.class, containerName); - - File yVarDir = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/home/y/var").toFile(); - if (yVarDir.exists()) { - DeleteOldAppData.deleteDirectories(yVarDir.getAbsolutePath(), 0, null); - } - - Path from = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/"); - if (!Files.exists(from)) { - logger.info("The application storage at " + from + " doesn't exist"); - return; - } - - Path to = Maintainer.pathInNodeAdminToNodeCleanup(containerName); - logger.info("Deleting application storage by moving it from " + from + " to " + to); - //TODO: move to maintenance JVM - Files.move(from, to); - } - - private void execute(PrefixLogger logger, String... params) { - try { - Process p = Runtime.getRuntime().exec(concatenateArrays(baseArguments, params)); - String output = IOUtils.readAll(new InputStreamReader(p.getInputStream())); - String errors = IOUtils.readAll(new InputStreamReader(p.getErrorStream())); - - if (! output.isEmpty()) logger.info(output); - if (! errors.isEmpty()) logger.error(errors); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private static String[] concatenateArrays(String[] ar1, String[] ar2) { - String[] concatenated = new String[ar1.length + ar2.length]; - System.arraycopy(ar1, 0, concatenated, 0, ar1.length); - System.arraycopy(ar2, 0, concatenated, ar1.length, ar2.length); - return concatenated; - } -}
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java new file mode 100644 index 00000000000..0dd1a24d93e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -0,0 +1,195 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance; + +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.node.admin.restapi.SecretAgentHandler; +import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; +import com.yahoo.vespa.hosted.node.maintenance.DeleteOldAppData; +import com.yahoo.vespa.hosted.node.maintenance.Maintainer; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * @author valerijf + */ +public class StorageMaintainer { + private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(StorageMaintainer.class); + private static final String[] baseArguments = {"sudo", "/home/y/libexec/vespa/node-admin/maintenance.sh"}; + private static final long intervalSec = 1000; + + private final Object monitor = new Object(); + + private Map<ContainerName, MetricsCache> metricsCacheByContainerName = new ConcurrentHashMap<>(); + private Random random = new Random(); + + public void updateDiskUsage(String hostname, ContainerName containerName) { + updateMetricsCacheForContainerIfNeeded(containerName); + + try { + PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName); + SecretAgentHandler secretAgentHandler = new SecretAgentHandler(); + secretAgentHandler.withDimension("host", hostname); + metricsCacheByContainerName.get(containerName).metrics.forEach(secretAgentHandler::withMetric); + + // First write to temp file, then move temp file to main file to achieve atomic write + Path metricsSharePath = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/metrics-share/disk.usage"); + Path metricsSharePathTemp = Paths.get(metricsSharePath.toString() + "_temp"); + Files.write(metricsSharePathTemp, secretAgentHandler.toJson().getBytes(StandardCharsets.UTF_8.name())); + + // Files.move() fails to move if target already exist, could do target.delete() first, but then it's no longer atomic + execute(logger, "mv", metricsSharePathTemp.toString(), metricsSharePath.toString()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private void updateMetricsCacheForContainerIfNeeded(ContainerName containerName) { + // Calculating disk usage is IO expensive operation and its value changes relatively slowly, we want to perform + // that calculation rarely. Additionally, we spread out the calculation for different containers by adding + // a random deviation. + if (metricsCacheByContainerName.containsKey(containerName) && + metricsCacheByContainerName.get(containerName).nextUpdateAt.isAfter(Instant.now())) return; + + long distributedSecs = (long) (intervalSec * (0.5 + random.nextDouble())); + MetricsCache metricsCache = new MetricsCache(Instant.now().plusSeconds(distributedSecs)); + + // Throttle to one disk usage calculation at a time. + synchronized (monitor) { + PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName); + File containerDir = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/home/").toFile(); + + try { + long used = getDiscUsedInBytes(containerDir); + metricsCache.metrics.put("node.disk.used", used); + } catch (Throwable e) { + logger.error("Problems during disk usage calculations: " + e.getMessage()); + } + } + + metricsCacheByContainerName.put(containerName, metricsCache); + } + + // Public for testing + long getDiscUsedInBytes(File path) throws IOException, InterruptedException { + final String[] command = {"du", "-xsk", path.toString()}; + + Process duCommand = new ProcessBuilder().command(command).start(); + if (!duCommand.waitFor(60, TimeUnit.SECONDS)) { + duCommand.destroy(); + throw new RuntimeException("Disk usage command timedout, aborting."); + } + String output = IOUtils.readAll(new InputStreamReader(duCommand.getInputStream())); + String error = IOUtils.readAll(new InputStreamReader(duCommand.getErrorStream())); + + if (! error.isEmpty()) { + throw new RuntimeException("Disk usage wrote to error log: " + error); + } + + String[] results = output.split("\t"); + if (results.length != 2) { + throw new RuntimeException("Result from disk usage command not as expected: " + output); + } + long diskUsageKB = Long.valueOf(results[0]); + + return diskUsageKB * 1024; + } + + + public void removeOldFilesFromNode(ContainerName containerName) { + PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName); + + String[] pathsToClean = {"/home/y/logs/elasticsearch2", "/home/y/logs/logstash2", + "/home/y/logs/daemontools_y", "/home/y/logs/nginx", "/home/y/logs/vespa"}; + for (String pathToClean : pathsToClean) { + File path = Maintainer.pathInNodeAdminFromPathInNode(containerName, pathToClean).toFile(); + if (path.exists()) { + DeleteOldAppData.deleteFiles(path.getAbsolutePath(), Duration.ofDays(3).getSeconds(), ".*\\.log\\..+", false); + DeleteOldAppData.deleteFiles(path.getAbsolutePath(), Duration.ofDays(3).getSeconds(), ".*QueryAccessLog.*", false); + } + } + + File logArchiveDir = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/home/y/logs/vespa/logarchive").toFile(); + if (logArchiveDir.exists()) { + DeleteOldAppData.deleteFiles(logArchiveDir.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false); + } + + File fileDistrDir = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/home/y/var/db/vespa/filedistribution").toFile(); + if (fileDistrDir.exists()) { + DeleteOldAppData.deleteFiles(fileDistrDir.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false); + } + + execute(logger, concatenateArrays(baseArguments, Maintainer.JOB_CLEAN_CORE_DUMPS)); + } + + public void cleanNodeAdmin() { + execute(NODE_ADMIN_LOGGER, concatenateArrays(baseArguments, Maintainer.JOB_DELETE_OLD_APP_DATA)); + execute(NODE_ADMIN_LOGGER, concatenateArrays(baseArguments, Maintainer.JOB_CLEAN_HOME)); + + File nodeAdminJDiskLogsPath = Maintainer.pathInNodeAdminFromPathInNode(new ContainerName("node-admin"), + "/home/y/logs/jdisc_core/").toFile(); + DeleteOldAppData.deleteFiles(nodeAdminJDiskLogsPath.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false); + } + + public void deleteContainerStorage(ContainerName containerName) throws IOException { + PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName); + + File yVarDir = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/home/y/var").toFile(); + if (yVarDir.exists()) { + DeleteOldAppData.deleteDirectories(yVarDir.getAbsolutePath(), 0, null); + } + + Path from = Maintainer.pathInNodeAdminFromPathInNode(containerName, "/"); + if (!Files.exists(from)) { + logger.info("The application storage at " + from + " doesn't exist"); + return; + } + + Path to = Maintainer.pathInNodeAdminToNodeCleanup(containerName); + logger.info("Deleting application storage by moving it from " + from + " to " + to); + //TODO: move to maintenance JVM + Files.move(from, to); + } + + private void execute(PrefixLogger logger, String... params) { + try { + Process p = Runtime.getRuntime().exec(params); + String output = IOUtils.readAll(new InputStreamReader(p.getInputStream())); + String errors = IOUtils.readAll(new InputStreamReader(p.getErrorStream())); + + if (! output.isEmpty()) logger.info(output); + if (! errors.isEmpty()) logger.error(errors); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private static String[] concatenateArrays(String[] ar1, String... ar2) { + String[] concatenated = new String[ar1.length + ar2.length]; + System.arraycopy(ar1, 0, concatenated, 0, ar1.length); + System.arraycopy(ar2, 0, concatenated, ar1.length, ar2.length); + return concatenated; + } + + private static class MetricsCache { + private final Instant nextUpdateAt; + private final Map<String, Object> metrics = new HashMap<>(); + + MetricsCache(Instant nextUpdateAt) { + this.nextUpdateAt = nextUpdateAt; + } + } +}
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 600f4b16931..aaebe1b3784 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -2,13 +2,17 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.collections.Pair; +import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.node.admin.maintenance.MaintenanceScheduler; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; +import com.yahoo.vespa.hosted.provision.Node; import java.io.IOException; import java.time.Duration; @@ -37,7 +41,7 @@ public class NodeAdminImpl implements NodeAdmin { private final Docker docker; private final Function<String, NodeAgent> nodeAgentFactory; - private final MaintenanceScheduler maintenanceScheduler; + private final StorageMaintainer storageMaintainer; private AtomicBoolean frozen = new AtomicBoolean(false); private final Map<String, NodeAgent> nodeAgents = new HashMap<>(); @@ -46,24 +50,52 @@ public class NodeAdminImpl implements NodeAdmin { private final int nodeAgentScanIntervalMillis; + private GaugeWrapper numberOfContainersInActiveState; + private GaugeWrapper numberOfContainersInLoadImageState; + private CounterWrapper numberOfUnhandledExceptionsInNodeAgent; + /** * @param docker interface to docker daemon and docker-related tasks * @param nodeAgentFactory factory for {@link NodeAgent} objects */ public NodeAdminImpl(final Docker docker, final Function<String, NodeAgent> nodeAgentFactory, - final MaintenanceScheduler maintenanceScheduler, int nodeAgentScanIntervalMillis) { + final StorageMaintainer storageMaintainer, int nodeAgentScanIntervalMillis, + final MetricReceiverWrapper metricReceiver) { this.docker = docker; this.nodeAgentFactory = nodeAgentFactory; - this.maintenanceScheduler = maintenanceScheduler; + this.storageMaintainer = storageMaintainer; this.nodeAgentScanIntervalMillis = nodeAgentScanIntervalMillis; + + this.numberOfContainersInActiveState = metricReceiver.declareGauge("nodes.state.active"); + this.numberOfContainersInLoadImageState = metricReceiver.declareGauge("nodes.image.loading"); + this.numberOfUnhandledExceptionsInNodeAgent = metricReceiver.declareCounter("nodes.unhandled_exceptions"); } public void refreshContainersToRun(final List<ContainerNodeSpec> containersToRun) { final List<Container> existingContainers = docker.getAllManagedContainers(); - maintenanceScheduler.cleanNodeAdmin(); + storageMaintainer.cleanNodeAdmin(); synchronizeNodeSpecsToNodeAgents(containersToRun, existingContainers); garbageCollectDockerImages(containersToRun); + + updateNodeAgentMetrics(); + } + + private void updateNodeAgentMetrics() { + int numberContainersInActive = 0; + int numberContainersWaitingImage = 0; + int numberOfNewUnhandledExceptions = 0; + + for (NodeAgent nodeAgent : nodeAgents.values()) { + Optional<ContainerNodeSpec> nodeSpec = nodeAgent.getContainerNodeSpec(); + if (nodeSpec.isPresent() && nodeSpec.get().nodeState == Node.State.active) numberContainersInActive++; + if (nodeAgent.isDownloadingImage()) numberContainersWaitingImage++; + numberOfNewUnhandledExceptions += nodeAgent.getAndResetNumberOfUnhandledExceptions(); + } + + numberOfContainersInActiveState.sample(numberContainersInActive); + numberOfContainersInLoadImageState.sample(numberContainersWaitingImage); + numberOfUnhandledExceptionsInNodeAgent.add(numberOfNewUnhandledExceptions); } public boolean freezeNodeAgentsAndCheckIfAllFrozen() { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java index fe6af0fdeec..50b29991527 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import java.util.Map; +import java.util.Optional; /** * Responsible for management of a single node over its lifecycle. @@ -55,5 +56,15 @@ public interface NodeAgent { /** * Returns the {@link ContainerNodeSpec} for this node agent. */ - ContainerNodeSpec getContainerNodeSpec(); + Optional<ContainerNodeSpec> getContainerNodeSpec(); + + /** + * Returns true if NodeAgent is waiting for an image download to finish + */ + boolean isDownloadingImage(); + + /** + * Returns and resets number of unhandled exceptions + */ + int getAndResetNumberOfUnhandledExceptions(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index fd59002edcf..d00068c2e58 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.maintenance.MaintenanceScheduler; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; @@ -18,6 +18,7 @@ import java.util.Date; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.ABSENT; @@ -45,7 +46,7 @@ public class NodeAgentImpl implements NodeAgent { private final NodeRepository nodeRepository; private final Orchestrator orchestrator; private final DockerOperations dockerOperations; - private final MaintenanceScheduler maintenanceScheduler; + private final StorageMaintainer storageMaintainer; private final Object monitor = new Object(); @@ -53,6 +54,7 @@ public class NodeAgentImpl implements NodeAgent { private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private long delaysBetweenEachTickMillis; + private int numberOfUnhandledException = 0; private Thread loopThread; @@ -72,12 +74,12 @@ public class NodeAgentImpl implements NodeAgent { final NodeRepository nodeRepository, final Orchestrator orchestrator, final DockerOperations dockerOperations, - final MaintenanceScheduler maintenanceScheduler) { + final StorageMaintainer storageMaintainer) { this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; this.hostname = hostName; this.dockerOperations = dockerOperations; - this.maintenanceScheduler = maintenanceScheduler; + this.storageMaintainer = storageMaintainer; this.logger = PrefixLogger.getNodeAgentLogger(NodeAgentImpl.class, NodeRepositoryImpl.containerNameFromHostName(hostName)); } @@ -138,7 +140,7 @@ public class NodeAgentImpl implements NodeAgent { throw new RuntimeException("Can not restart a node agent."); } loopThread = new Thread(this::loop); - loopThread.setName("loop-" + hostname.toString()); + loopThread.setName("loop-" + hostname); loopThread.start(); } @@ -232,7 +234,7 @@ public class NodeAgentImpl implements NodeAgent { imageBeingDownloaded = nodeSpec.wantedDockerImage.get(); // Create a signalWorkToBeDone when download is finished. dockerOperations.scheduleDownloadOfImage(nodeSpec, this::signalWorkToBeDone); - } else { + } else if (imageBeingDownloaded != null) { // Image was downloading, but now its ready imageBeingDownloaded = null; } } @@ -272,6 +274,7 @@ public class NodeAgentImpl implements NodeAgent { try { tick(); } catch (Exception e) { + numberOfUnhandledException++; logger.error("Unhandled exception, ignoring.", e); addDebugMessage(e.getMessage()); } catch (Throwable t) { @@ -303,7 +306,7 @@ public class NodeAgentImpl implements NodeAgent { removeContainerIfNeededUpdateContainerState(nodeSpec); break; case active: - maintenanceScheduler.removeOldFilesFromNode(nodeSpec.containerName); + storageMaintainer.removeOldFilesFromNode(nodeSpec.containerName); scheduleDownLoadIfNeeded(nodeSpec); if (imageBeingDownloaded != null) { addDebugMessage("Waiting for image to download " + imageBeingDownloaded.asString()); @@ -326,17 +329,18 @@ public class NodeAgentImpl implements NodeAgent { updateNodeRepoWithCurrentAttributes(nodeSpec); logger.info("Call resume against Orchestrator"); orchestrator.resume(nodeSpec.hostname); + storageMaintainer.updateDiskUsage(nodeSpec.hostname, nodeSpec.containerName); break; case inactive: - maintenanceScheduler.removeOldFilesFromNode(nodeSpec.containerName); + storageMaintainer.removeOldFilesFromNode(nodeSpec.containerName); removeContainerIfNeededUpdateContainerState(nodeSpec); break; case provisioned: case dirty: - maintenanceScheduler.removeOldFilesFromNode(nodeSpec.containerName); + storageMaintainer.removeOldFilesFromNode(nodeSpec.containerName); removeContainerIfNeededUpdateContainerState(nodeSpec); logger.info("State is " + nodeSpec.nodeState + ", will delete application storage and mark node as ready"); - maintenanceScheduler.deleteContainerStorage(nodeSpec.containerName); + storageMaintainer.deleteContainerStorage(nodeSpec.containerName); updateNodeRepoAndMarkNodeAsReady(nodeSpec); break; case parked: @@ -348,9 +352,21 @@ public class NodeAgentImpl implements NodeAgent { } } - public ContainerNodeSpec getContainerNodeSpec() { + public Optional<ContainerNodeSpec> getContainerNodeSpec() { synchronized (monitor) { - return lastNodeSpec; + return Optional.ofNullable(lastNodeSpec); } } + + @Override + public boolean isDownloadingImage() { + return imageBeingDownloaded != null; + } + + @Override + public int getAndResetNumberOfUnhandledExceptions() { + int temp = numberOfUnhandledException; + numberOfUnhandledException = 0; + return temp; + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java index a8439bfc8cc..b91c2411d95 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.orchestrator; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; @@ -22,8 +23,7 @@ import java.util.Set; */ public class OrchestratorImpl implements Orchestrator { private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(OrchestratorImpl.class); - // TODO: Figure out the port dynamically. - static final int HARDCODED_ORCHESTRATOR_PORT = 19071; + static final int WEB_SERVICE_PORT = Defaults.getDefaults().vespaWebServicePort(); // TODO: Find a way to avoid duplicating this (present in orchestrator's services.xml also). private static final String ORCHESTRATOR_PATH_PREFIX = "/orchestrator"; static final String ORCHESTRATOR_PATH_PREFIX_HOST_API @@ -53,7 +53,7 @@ public class OrchestratorImpl implements Orchestrator { try { final UpdateHostResponse updateHostResponse = requestExecutor.put( ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended", - HARDCODED_ORCHESTRATOR_PORT, + WEB_SERVICE_PORT, Optional.empty(), /* body */ UpdateHostResponse.class); return updateHostResponse.reason() == null; @@ -72,7 +72,7 @@ public class OrchestratorImpl implements Orchestrator { try { final BatchOperationResult batchOperationResult = requestExecutor.put( ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - HARDCODED_ORCHESTRATOR_PORT, + WEB_SERVICE_PORT, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class); return batchOperationResult.getFailureReason(); @@ -89,7 +89,7 @@ public class OrchestratorImpl implements Orchestrator { try { final UpdateHostResponse batchOperationResult = requestExecutor.delete( ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName + "/suspended", - HARDCODED_ORCHESTRATOR_PORT, + WEB_SERVICE_PORT, UpdateHostResponse.class); return batchOperationResult.reason() == null; } catch (ConfigServerHttpRequestExecutor.NotFoundException n) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java index b0e07e03eea..f00b6bb828c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.provider; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; /** @@ -10,4 +11,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; */ public interface ComponentsProvider { NodeAdminStateUpdater getNodeAdminStateUpdater(); + + MetricReceiverWrapper getMetricReceiverWrapper(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java index 19988959691..aa01fd602c4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.provider; -import com.yahoo.vespa.hosted.node.admin.maintenance.MaintenanceScheduler; -import com.yahoo.vespa.hosted.node.admin.maintenance.MaintenanceSchedulerImpl; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; @@ -26,18 +27,18 @@ import java.util.function.Function; */ public class ComponentsProviderImpl implements ComponentsProvider { - private final Docker docker; private final NodeAdminStateUpdater nodeAdminStateUpdater; + private final MetricReceiverWrapper metricReceiverWrapper; private static final long INITIAL_SCHEDULER_DELAY_MILLIS = 1; - private static final int NODE_AGENT_SCAN_INTERVAL_MILLIS = 60000; - private static final int HARDCODED_NODEREPOSITORY_PORT = 19071; + private static final int NODE_AGENT_SCAN_INTERVAL_MILLIS = 30000; + private static final int WEB_SERVICE_PORT = Defaults.getDefaults().vespaWebServicePort(); private static final String ENV_HOSTNAME = "HOSTNAME"; // We only scan for new nodes within a host every 5 minutes. This is only if new nodes are added or removed // which happens rarely. Changes of apps running etc it detected by the NodeAgent. private static final int NODE_ADMIN_STATE_INTERVAL_MILLIS = 5 * 60000; - public ComponentsProviderImpl(final Docker docker) { - this.docker = docker; + + public ComponentsProviderImpl(final Docker docker, final MetricReceiverWrapper metricReceiver) { String baseHostName = java.util.Optional.ofNullable(System.getenv(ENV_HOSTNAME)) .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset")); @@ -45,18 +46,26 @@ public class ComponentsProviderImpl implements ComponentsProvider { Set<String> configServerHosts = environment.getConfigServerHosts(); Orchestrator orchestrator = new OrchestratorImpl(configServerHosts); - NodeRepository nodeRepository = new NodeRepositoryImpl(configServerHosts, HARDCODED_NODEREPOSITORY_PORT, baseHostName); - MaintenanceScheduler maintenanceScheduler = new MaintenanceSchedulerImpl(); + NodeRepository nodeRepository = new NodeRepositoryImpl(configServerHosts, WEB_SERVICE_PORT, baseHostName); + StorageMaintainer storageMaintainer = new StorageMaintainer(); final Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepository, - orchestrator, new DockerOperationsImpl(docker, environment), maintenanceScheduler); - final NodeAdmin nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory, maintenanceScheduler, NODE_AGENT_SCAN_INTERVAL_MILLIS); + orchestrator, new DockerOperationsImpl(docker, environment), storageMaintainer); + final NodeAdmin nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory, storageMaintainer, + NODE_AGENT_SCAN_INTERVAL_MILLIS, metricReceiver); nodeAdminStateUpdater = new NodeAdminStateUpdater( nodeRepository, nodeAdmin, INITIAL_SCHEDULER_DELAY_MILLIS, NODE_ADMIN_STATE_INTERVAL_MILLIS, orchestrator, baseHostName); + + metricReceiverWrapper = metricReceiver; } @Override public NodeAdminStateUpdater getNodeAdminStateUpdater() { return nodeAdminStateUpdater; } + + @Override + public MetricReceiverWrapper getMetricReceiverWrapper() { + return metricReceiverWrapper; + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java index 2ce4151f497..ba083b5e593 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java @@ -7,12 +7,15 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; +import com.yahoo.net.HostName; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProvider; import javax.ws.rs.core.MediaType; import java.io.IOException; import java.io.OutputStream; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.concurrent.Executor; @@ -30,12 +33,14 @@ import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; */ public class RestApiHandler extends LoggingRequestHandler{ - private final NodeAdminStateUpdater refresher; private final static ObjectMapper objectMapper = new ObjectMapper(); + private final NodeAdminStateUpdater refresher; + private final MetricReceiverWrapper metricReceiverWrapper; public RestApiHandler(Executor executor, AccessLog accessLog, ComponentsProvider componentsProvider) { super(executor, accessLog); this.refresher = componentsProvider.getNodeAdminStateUpdater(); + this.metricReceiverWrapper = componentsProvider.getMetricReceiverWrapper(); } @Override @@ -47,16 +52,30 @@ public class RestApiHandler extends LoggingRequestHandler{ return handlePut(request); } return new SimpleResponse(400, "Only PUT and GET are implemented."); - } private HttpResponse handleGet(HttpRequest request) { String path = request.getUri().getPath(); if (path.endsWith("/info")) { + return new SimpleObjectResponse(200, refresher.getDebugPage()); + } + + if (path.endsWith("/metrics")) { + SecretAgentHandler secretAgentHandler = new SecretAgentHandler(); + secretAgentHandler.withDimension("host", HostName.getLocalhost()); + metricReceiverWrapper.getLatestMetrics().forEach(secretAgentHandler::withMetric); + return new HttpResponse(200) { @Override + public String getContentType() { + return MediaType.APPLICATION_JSON; + } + + @Override public void render(OutputStream outputStream) throws IOException { - objectMapper.writeValue(outputStream, refresher.getDebugPage()); + try (PrintStream printStream = new PrintStream(outputStream)) { + printStream.write(secretAgentHandler.toJson().getBytes(StandardCharsets.UTF_8.name())); + } } }; } @@ -84,10 +103,9 @@ public class RestApiHandler extends LoggingRequestHandler{ } private static class SimpleResponse extends HttpResponse { - private final String jsonMessage; - public SimpleResponse(int code, String message) { + SimpleResponse(int code, String message) { super(code); ObjectNode objectNode = objectMapper.createObjectNode(); objectNode.put("jsonMessage", message); @@ -105,4 +123,22 @@ public class RestApiHandler extends LoggingRequestHandler{ } } + private static class SimpleObjectResponse extends HttpResponse { + private final Object response; + + SimpleObjectResponse(int status, Object response) { + super(status); + this.response = response; + } + + @Override + public String getContentType() { + return MediaType.APPLICATION_JSON; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + objectMapper.writeValue(outputStream, response); + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/SecretAgentHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/SecretAgentHandler.java new file mode 100644 index 00000000000..e266d18eece --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/SecretAgentHandler.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.restapi; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Collects last value from all the previously declared counters/gauges and genereates a map + * structure that can be converted to secret-agent JSON message + * + * @author valerijf + */ +public class SecretAgentHandler { + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private static final String applicationName = "docker"; + private final Map<String, Object> dimensions = new HashMap<>(); + private final Map<String, Object> metrics = new HashMap<>(); + + public SecretAgentHandler withDimension(String name, Object value) { + dimensions.put(name, value); + return this; + } + + public SecretAgentHandler withMetric(String name, Object value) { + metrics.put(name, value); + return this; + } + + public String toJson() throws JsonProcessingException { + Map<String, Object> report = new LinkedHashMap<>(); + report.put("application", applicationName); + report.put("timestamp", System.currentTimeMillis() / 1000); + report.put("dimensions", dimensions); + report.put("metrics", metrics); + + return objectMapper.writeValueAsString(report); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java index 2984bbd563b..67e0e0c0552 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; @@ -21,18 +23,24 @@ import java.util.function.Function; public class ComponentsProviderWithMocks implements ComponentsProvider { static final CallOrderVerifier callOrder = new CallOrderVerifier(); static final NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrder); - static final MaintenanceSchedulerMock maintenanceSchedulerMock = new MaintenanceSchedulerMock(callOrder); + static final StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrder); static final OrchestratorMock orchestratorMock = new OrchestratorMock(callOrder); static final Docker dockerMock = new DockerMock(callOrder); private Environment environment = new Environment(); private final Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, new DockerOperationsImpl(dockerMock, environment), maintenanceSchedulerMock); - private NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100); + private NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, + new MetricReceiverWrapper(MetricReceiver.nullImplementation)); @Override public NodeAdminStateUpdater getNodeAdminStateUpdater() { return new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 5, orchestratorMock, "localhost"); } + + @Override + public MetricReceiverWrapper getMetricReceiverWrapper() { + return null; + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java index 6948448e336..7ccbeb23166 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java @@ -1,9 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; @@ -39,7 +41,7 @@ public class DockerFailTest { @Before public void before() throws InterruptedException, UnknownHostException { callOrder = new CallOrderVerifier(); - MaintenanceSchedulerMock maintenanceSchedulerMock = new MaintenanceSchedulerMock(callOrder); + StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrder); OrchestratorMock orchestratorMock = new OrchestratorMock(callOrder); NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrder); dockerMock = new DockerMock(callOrder); @@ -50,7 +52,8 @@ public class DockerFailTest { Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, new DockerOperationsImpl(dockerMock, environment), maintenanceSchedulerMock); - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100); + NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, + new MetricReceiverWrapper(MetricReceiver.nullImplementation)); initialContainerNodeSpec = new ContainerNodeSpec( "hostName", diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index f57f6df422e..81c8965a55c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -1,8 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; @@ -41,7 +43,7 @@ public class MultiDockerTest { @Before public void before() throws InterruptedException, UnknownHostException { callOrder = new CallOrderVerifier(); - MaintenanceSchedulerMock maintenanceSchedulerMock = new MaintenanceSchedulerMock(callOrder); + StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrder); OrchestratorMock orchestratorMock = new OrchestratorMock(callOrder); nodeRepositoryMock = new NodeRepoMock(callOrder); dockerMock = new DockerMock(callOrder); @@ -52,7 +54,8 @@ public class MultiDockerTest { Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, new DockerOperationsImpl(dockerMock, environment), maintenanceSchedulerMock); - nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100); + nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, + new MetricReceiverWrapper(MetricReceiver.nullImplementation)); updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java index ad45f3ef2f1..e89ebafab8f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; @@ -46,7 +48,7 @@ public class NodeStateTest { @Before public void before() throws InterruptedException, UnknownHostException { callOrder = new CallOrderVerifier(); - MaintenanceSchedulerMock maintenanceSchedulerMock = new MaintenanceSchedulerMock(callOrder); + StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrder); OrchestratorMock orchestratorMock = new OrchestratorMock(callOrder); nodeRepositoryMock = new NodeRepoMock(callOrder); dockerMock = new DockerMock(callOrder); @@ -57,7 +59,8 @@ public class NodeStateTest { Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, new DockerOperationsImpl(dockerMock, environment), maintenanceSchedulerMock); - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100); + NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, + new MetricReceiverWrapper(MetricReceiver.nullImplementation)); initialContainerNodeSpec = new ContainerNodeSpec( "host1", diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java index 9af0d7a56a4..a1db7eb9b97 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; @@ -37,7 +39,7 @@ public class ResumeTest { public void test() throws InterruptedException, UnknownHostException { CallOrderVerifier callOrder = new CallOrderVerifier(); NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrder); - MaintenanceSchedulerMock maintenanceSchedulerMock = new MaintenanceSchedulerMock(callOrder); + StorageMaintainerMock maintenanceSchedulerMock = new StorageMaintainerMock(callOrder); OrchestratorMock orchestratorMock = new OrchestratorMock(callOrder); DockerMock dockerMock = new DockerMock(callOrder); @@ -47,7 +49,8 @@ public class ResumeTest { Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, new DockerOperationsImpl(dockerMock, environment), maintenanceSchedulerMock); - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100); + NodeAdmin nodeAdmin = new NodeAdminImpl(dockerMock, nodeAgentFactory, maintenanceSchedulerMock, 100, + new MetricReceiverWrapper(MetricReceiver.nullImplementation)); nodeRepositoryMock.addContainerNodeSpec(new ContainerNodeSpec( "host1", diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MaintenanceSchedulerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index 30ddc71f546..7356c2c34b9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MaintenanceSchedulerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -2,28 +2,30 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.node.admin.maintenance.MaintenanceScheduler; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import java.io.IOException; /** * @author valerijf */ -public class MaintenanceSchedulerMock implements MaintenanceScheduler { +public class StorageMaintainerMock extends StorageMaintainer { private final CallOrderVerifier callOrder; - public MaintenanceSchedulerMock(CallOrderVerifier callOrder) { + public StorageMaintainerMock(CallOrderVerifier callOrder) { this.callOrder = callOrder; } @Override - public void removeOldFilesFromNode(ContainerName containerName) { + public void updateDiskUsage(String hostname, ContainerName containerName) { + } + @Override + public void removeOldFilesFromNode(ContainerName containerName) { } @Override public void cleanNodeAdmin() { - } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java new file mode 100644 index 00000000000..c1603a7535e --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -0,0 +1,30 @@ +package com.yahoo.vespa.hosted.node.admin.maintenance; + +import com.yahoo.vespa.hosted.node.maintenance.DeleteOldAppDataTest; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; + +import static org.junit.Assert.*; + +/** + * @author dybis + */ +public class StorageMaintainerTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testDiskUsed() throws IOException, InterruptedException { + int writeSize = 10000; + DeleteOldAppDataTest.writeNBytesToFile(folder.newFile(), writeSize); + + StorageMaintainer storageMaintainer = new StorageMaintainer(); + long usedBytes = storageMaintainer.getDiscUsedInBytes(folder.getRoot()); + if (usedBytes * 4 < writeSize || usedBytes > writeSize * 4) + fail("Used bytes is " + usedBytes + ", but wrote " + writeSize + " bytes, not even close."); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index e1dac0844c3..5af24e71c3d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -2,12 +2,14 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.collections.Pair; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.node.admin.maintenance.MaintenanceScheduler; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.provision.Node; @@ -48,9 +50,10 @@ public class NodeAdminImplTest { public void nodeAgentsAreProperlyLifeCycleManaged() throws Exception { final Docker docker = mock(Docker.class); final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); - final MaintenanceScheduler maintenanceScheduler = mock(MaintenanceScheduler.class); + final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - final NodeAdminImpl nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory, maintenanceScheduler, 100); + final NodeAdminImpl nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory, storageMaintainer, 100, + new MetricReceiverWrapper(MetricReceiver.nullImplementation)); final NodeAgent nodeAgent1 = mock(NodeAgentImpl.class); final NodeAgent nodeAgent2 = mock(NodeAgentImpl.class); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 871032005aa..1e748b74ea9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.maintenance.MaintenanceScheduler; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; @@ -38,9 +38,9 @@ public class NodeAgentImplTest { private final DockerOperations dockerOperations = mock(DockerOperations.class); private final NodeRepository nodeRepository = mock(NodeRepository.class); private final Orchestrator orchestrator = mock(Orchestrator.class); - private final MaintenanceScheduler maintenanceScheduler = mock(MaintenanceScheduler.class); + private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - private final NodeAgentImpl nodeAgent = new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, maintenanceScheduler); + private final NodeAgentImpl nodeAgent = new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, storageMaintainer); @Test public void upToDateContainerIsUntouched() throws Exception { @@ -262,8 +262,8 @@ public class NodeAgentImplTest { nodeAgent.tick(); - final InOrder inOrder = inOrder(maintenanceScheduler, dockerOperations); - inOrder.verify(maintenanceScheduler, times(1)).removeOldFilesFromNode(eq(containerName)); + final InOrder inOrder = inOrder(storageMaintainer, dockerOperations); + inOrder.verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(containerName)); inOrder.verify(dockerOperations, times(1)).removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any()); verify(orchestrator, never()).resume(any(String.class)); @@ -294,10 +294,10 @@ public class NodeAgentImplTest { nodeAgent.tick(); - final InOrder inOrder = inOrder(maintenanceScheduler, dockerOperations, nodeRepository); - inOrder.verify(maintenanceScheduler, times(1)).removeOldFilesFromNode(eq(containerName)); + final InOrder inOrder = inOrder(storageMaintainer, dockerOperations, nodeRepository); + inOrder.verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(containerName)); inOrder.verify(dockerOperations, times(1)).removeContainerIfNeeded(eq(nodeSpec), eq(hostName), any()); - inOrder.verify(maintenanceScheduler, times(1)).deleteContainerStorage(eq(containerName)); + inOrder.verify(storageMaintainer, times(1)).deleteContainerStorage(eq(containerName)); inOrder.verify(nodeRepository, times(1)).markAsReady(eq(hostName)); verify(dockerOperations, never()).startContainerIfNeeded(any()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java index 761aa1fad53..39af637a45a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java @@ -34,7 +34,7 @@ public class OrchestratorImplTest { public void testSuspendCall() { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", - OrchestratorImpl.HARDCODED_ORCHESTRATOR_PORT, + OrchestratorImpl.WEB_SERVICE_PORT, Optional.empty(), UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, null)); @@ -47,7 +47,7 @@ public class OrchestratorImplTest { public void testSuspendCallWithFailureReason() { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", - OrchestratorImpl.HARDCODED_ORCHESTRATOR_PORT, + OrchestratorImpl.WEB_SERVICE_PORT, Optional.empty(), UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "service", "fail"))); @@ -87,7 +87,7 @@ public class OrchestratorImplTest { public void testResumeCall() { when(requestExecutor.delete( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", - OrchestratorImpl.HARDCODED_ORCHESTRATOR_PORT, + OrchestratorImpl.WEB_SERVICE_PORT, UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, null)); @@ -99,7 +99,7 @@ public class OrchestratorImplTest { public void testResumeCallWithFailureReason() { when(requestExecutor.delete( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", - OrchestratorImpl.HARDCODED_ORCHESTRATOR_PORT, + OrchestratorImpl.WEB_SERVICE_PORT, UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "service", "fail"))); @@ -140,7 +140,7 @@ public class OrchestratorImplTest { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - OrchestratorImpl.HARDCODED_ORCHESTRATOR_PORT, + OrchestratorImpl.WEB_SERVICE_PORT, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class )).thenReturn(BatchOperationResult.successResult()); @@ -157,7 +157,7 @@ public class OrchestratorImplTest { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - OrchestratorImpl.HARDCODED_ORCHESTRATOR_PORT, + OrchestratorImpl.WEB_SERVICE_PORT, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class )).thenReturn(new BatchOperationResult(failureReason)); @@ -174,7 +174,7 @@ public class OrchestratorImplTest { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - OrchestratorImpl.HARDCODED_ORCHESTRATOR_PORT, + OrchestratorImpl.WEB_SERVICE_PORT, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class )).thenThrow(new RuntimeException(exceptionMessage)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/restapi/SecretAgentHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/restapi/SecretAgentHandlerTest.java new file mode 100644 index 00000000000..bd9b81bb17a --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/restapi/SecretAgentHandlerTest.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.restapi; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.Test; + +import java.util.regex.Pattern; + +import static org.hamcrest.Matchers.matchesPattern; +import static org.junit.Assert.*; + +/** + * @author valerijf + */ +public class SecretAgentHandlerTest { + @Test + public void testSecretAgentFormat() throws JsonProcessingException { + SecretAgentHandler secretAgentHandler = new SecretAgentHandler(); + secretAgentHandler + .withDimension("host", "host.name.test.yahoo.com") + .withDimension("dimention", 6) + .withMetric("runtime", 0.0254) + .withMetric("memory", 321415L); + + String expectedJson = Pattern.quote("{\"application\":\"docker\",\"timestamp\":") + + "[0-9]{10}" + // The timestamp is (currently) 10 digit long numbe, update to 11 on 20/11/2286 + Pattern.quote(",\"dimensions\":{\"host\":\"host.name.test.yahoo.com\",\"dimention\":6},\"metrics\":{\"memory\":321415,\"runtime\":0.0254}}"); + + assertThat(secretAgentHandler.toJson(), matchesPattern(expectedJson)); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppDataTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppDataTest.java index 84922852365..462216ea827 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppDataTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/maintenance/DeleteOldAppDataTest.java @@ -1,15 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.maintenance; -import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; -import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; import java.time.Duration; import java.util.Arrays; @@ -157,13 +156,13 @@ public class DeleteOldAppDataTest { initSubDirectories(); File temp1 = new File(folder.getRoot(), "small_file"); - writeNBytesToFiles(temp1, 50); + writeNBytesToFile(temp1, 50); File temp2 = new File(folder.getRoot(), "some_file"); - writeNBytesToFiles(temp2, 20); + writeNBytesToFile(temp2, 20); File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file"); - writeNBytesToFiles(temp3, 75); + writeNBytesToFile(temp3, 75); DeleteOldAppData.deleteFilesLargerThan(folder.getRoot(), 10); @@ -238,9 +237,7 @@ public class DeleteOldAppDataTest { return total; } - private static void writeNBytesToFiles(File file, int nBytes) throws IOException { - try (FileWriter writer = new FileWriter(file)) { - writer.write(StringUtils.repeat("0", nBytes)); - } + public static void writeNBytesToFile(File file, int nBytes) throws IOException { + Files.write(file.toPath(), new byte[nBytes]); } } |