diff options
76 files changed, 0 insertions, 5387 deletions
diff --git a/node-maintainer/OWNERS b/node-maintainer/OWNERS deleted file mode 100644 index e030acdbc5b..00000000000 --- a/node-maintainer/OWNERS +++ /dev/null @@ -1 +0,0 @@ -freva diff --git a/node-maintainer/README.md b/node-maintainer/README.md deleted file mode 100644 index 33df558effb..00000000000 --- a/node-maintainer/README.md +++ /dev/null @@ -1,21 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Node Admin Maintenance - -Executes maintenance jobs, such as deleting old logs, processing and reporting coredumps, on behalf of node-admin. -Node admin maintenance runs as a separate JVM from node-admin to make it possible to run it as root if needed. - -## Node Verification -Node verification for both hardware and spec. Hardware is verified by performing different benchmarking tasks, -while spec is verified by comparing information reported by the OS with the spec from node repository. - -### Execute examples -Spec verification and hardware benchmarks must be executed with config server host name as parameter - -SpecVerifier: -- sudo java -cp node-maintainer-jar-with-dependencies.jar com.yahoo.vespa.hosted.node.verification.spec.SpecVerifier cfg.1.hostname,cfg.2.hostname,... - -HardwareBenchmarker: -- sudo java -cp node-maintainer-jar-with-dependencies.jar com.yahoo.vespa.hosted.node.verification.hardware.HardwareBenchmarker cfg.1.hostname,cfg.2.hostname,... - -In "verification" you can find a TODO file where we have listed things we did not have time to implement. -Both programs have README explaining closer what it does. diff --git a/node-maintainer/pom.xml b/node-maintainer/pom.xml deleted file mode 100644 index 3c51bdc05be..00000000000 --- a/node-maintainer/pom.xml +++ /dev/null @@ -1,121 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - - <artifactId>node-maintainer</artifactId> - <version>7-SNAPSHOT</version> - <packaging>jar</packaging> - <name>${project.artifactId}</name> - - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>defaults</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - <dependency> - <groupId>io.airlift</groupId> - <artifactId>airline</artifactId> - <version>0.6</version> - </dependency> - - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-junit</artifactId> - <version>2.0.0.0</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>2.8.3</version> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <environmentVariables> - <VESPA_HOME>/opt/vespa</VESPA_HOME> - </environmentVariables> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java deleted file mode 100644 index fef5a695db6..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import com.yahoo.collections.Pair; -import com.yahoo.system.ProcessExecuter; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - -/** - * Takes in a compressed (lz4) or uncompressed core dump and collects relevant metadata. - * - * @author freva - */ -public class CoreCollector { - private static final String GDB_PATH = getDefaults().underVespaHome("bin64/gdb"); - private static final String LZ4_PATH = getDefaults().underVespaHome("bin64/lz4"); - private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$"); - private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'"); - private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'"); - private static final Pattern TOTAL_MEMORY_PATTERN = Pattern.compile("^MemTotal:\\s*(?<totalMem>\\d+) kB$", Pattern.MULTILINE); - - private static final Logger logger = Logger.getLogger(CoreCollector.class.getName()); - private final ProcessExecuter processExecuter; - - public CoreCollector(ProcessExecuter processExecuter) { - this.processExecuter = processExecuter; - } - - List<String> readInstallState(Path installStatePath) throws IOException { - Pair<Integer, String> result = processExecuter.exec(new String[]{"cat", installStatePath.toString()}); - - if (result.getFirst() != 0) { - throw new RuntimeException("Failed to read install state file at: " + installStatePath + ", result: " + result); - } - return Arrays.asList(result.getSecond().split("\n")); - } - - List<String> readRpmPackages() throws IOException { - Pair<Integer, String> result = processExecuter.exec(new String[]{"rpm", "-qa"}); - - if (result.getFirst() != 0) { - throw new RuntimeException("Failed to read RPM packages " + result); - } - return Arrays.asList(result.getSecond().split("\n")); - } - - Path readBinPathFallback(Path coredumpPath) throws IOException { - String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; - String[] wrappedCommand = new String[] {"/bin/sh", "-c", command}; - Pair<Integer, String> result = processExecuter.exec(wrappedCommand); - - Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getSecond()); - if (! matcher.find()) { - throw new RuntimeException(String.format("Failed to extract binary path from GDB, result: %s, command: %s", - result, Arrays.toString(wrappedCommand))); - } - return Paths.get(matcher.group("path").split(" ")[0]); - } - - Path readBinPath(Path coredumpPath) throws IOException { - String[] command = new String[] {"file", coredumpPath.toString()}; - try { - Pair<Integer, String> result = processExecuter.exec(command); - - if (result.getFirst() != 0) { - throw new RuntimeException("file command failed with " + result); - } - - Matcher execfnMatcher = EXECFN_PATH_PATTERN.matcher(result.getSecond()); - if (execfnMatcher.find()) { - return Paths.get(execfnMatcher.group("path").split(" ")[0]); - } - - Matcher fromMatcher = FROM_PATH_PATTERN.matcher(result.getSecond()); - if (fromMatcher.find()) { - return Paths.get(fromMatcher.group("path").split(" ")[0]); - } - } catch (Throwable e) { - logger.log(Level.WARNING, String.format("Failed getting bin path, command: %s. " + - "Trying fallback instead", Arrays.toString(command)), e); - } - - return readBinPathFallback(coredumpPath); - } - - List<String> readBacktrace(Path coredumpPath, Path binPath, boolean allThreads) throws IOException { - String threads = allThreads ? "thread apply all bt" : "bt"; - String[] command = new String[]{GDB_PATH, "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; - Pair<Integer, String> result = processExecuter.exec(command); - if (result.getFirst() != 0) { - throw new RuntimeException("Failed to read backtrace " + result + ", Command: " + Arrays.toString(command)); - } - return Arrays.asList(result.getSecond().split("\n")); - } - - Map<String, Object> collect(Path coredumpPath, Optional<Path> installStatePath) { - Map<String, Object> data = new LinkedHashMap<>(); - try { - coredumpPath = compressCoredump(coredumpPath); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed compressing/decompressing core dump", e); - } - - try { - Path binPath = readBinPath(coredumpPath); - - data.put("bin_path", binPath.toString()); - data.put("backtrace", readBacktrace(coredumpPath, binPath, false)); - data.put("backtrace_all_threads", readBacktrace(coredumpPath, binPath, true)); - } catch (Throwable e) { - logger.log(Level.WARNING, "Failed to extract backtrace", e); - } - - installStatePath.ifPresent(installState -> { - try { - data.put("install_state", readInstallState(installState)); - } catch (Exception e) { - logger.log(Level.WARNING, "Failed to read install state", e); - } - - try { - data.put("rpm_packages", readRpmPackages()); - } catch (Exception e) { - logger.log(Level.WARNING, "Failed to read RPM packages", e); - } - }); - - try { - deleteDecompressedCoredump(coredumpPath); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to delete decompressed core dump", e); - } - return data; - } - - - /** - * This method will either compress or decompress the core dump if the input path is to a decompressed or - * compressed core dump, respectively. - * - * @return Path to the decompressed core dump - */ - private Path compressCoredump(Path coredumpPath) throws IOException { - if (! coredumpPath.toString().endsWith(".lz4")) { - processExecuter.exec( - new String[]{LZ4_PATH, "-f", coredumpPath.toString(), coredumpPath.toString() + ".lz4"}); - return coredumpPath; - - } else { - if (!diskSpaceAvailable(coredumpPath)) { - throw new RuntimeException("Not decompressing " + coredumpPath + " due to not enough disk space available"); - } - - Path decompressedPath = Paths.get(coredumpPath.toString().replaceFirst("\\.lz4$", "")); - Pair<Integer, String> result = processExecuter.exec( - new String[] {LZ4_PATH, "-f", "-d", coredumpPath.toString(), decompressedPath.toString()}); - if (result.getFirst() != 0) { - throw new RuntimeException("Failed to decompress file " + coredumpPath + ": " + result); - } - return decompressedPath; - } - } - - /** - * Delete the core dump unless: - * - The file is compressed - * - There is no compressed file (i.e. it was not decompressed in the first place) - */ - void deleteDecompressedCoredump(Path coredumpPath) throws IOException { - if (! coredumpPath.toString().endsWith(".lz4") && Paths.get(coredumpPath.toString() + ".lz4").toFile().exists()) { - Files.delete(coredumpPath); - } - } - - private boolean diskSpaceAvailable(Path path) throws IOException { - // TODO: If running inside container, check against container memory size, not for the enitre host - String memInfo = new String(Files.readAllBytes(Paths.get("/proc/meminfo"))); - return path.toFile().getFreeSpace() > parseTotalMemorySize(memInfo); - } - - int parseTotalMemorySize(String memInfo) { - Matcher matcher = TOTAL_MEMORY_PATTERN.matcher(memInfo); - if (!matcher.find()) throw new RuntimeException("Could not parse meminfo: " + memInfo); - return Integer.valueOf(matcher.group("totalMem")); - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java deleted file mode 100644 index 0c118e2dc6c..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Finds coredumps, collects metadata and reports them - * - * @author freva - */ -class CoredumpHandler { - - static final String PROCESSING_DIRECTORY_NAME = "processing"; - static final String METADATA_FILE_NAME = "metadata.json"; - - private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); - private final ObjectMapper objectMapper = new ObjectMapper(); - - private final CloseableHttpClient httpClient; - private final CoreCollector coreCollector; - private final Path coredumpsPath; - private final Path doneCoredumpsPath; - private final Map<String, Object> nodeAttributes; - private final Optional<Path> installStatePath; - private final String feedEndpoint; - - public CoredumpHandler(CloseableHttpClient httpClient, CoreCollector coreCollector, Path coredumpsPath, Path doneCoredumpsPath, - Map<String, Object> nodeAttributes, Optional<Path> installStatePath, String feedEndpoint) { - this.httpClient = httpClient; - this.coreCollector = coreCollector; - this.coredumpsPath = coredumpsPath; - this.doneCoredumpsPath = doneCoredumpsPath; - this.nodeAttributes = nodeAttributes; - this.installStatePath = installStatePath; - this.feedEndpoint = feedEndpoint; - } - - public void processAll() throws IOException { - removeJavaCoredumps(); - handleNewCoredumps(); - removeOldCoredumps(); - } - - private void removeJavaCoredumps() throws IOException { - if (! coredumpsPath.toFile().isDirectory()) return; - FileHelper.deleteFiles(coredumpsPath, Duration.ZERO, Optional.of("^java_pid.*\\.hprof$"), false); - } - - private void removeOldCoredumps() throws IOException { - if (! doneCoredumpsPath.toFile().isDirectory()) return; - FileHelper.deleteDirectories(doneCoredumpsPath, Duration.ofDays(10), Optional.empty()); - } - - private void handleNewCoredumps() { - Path processingCoredumps = enqueueCoredumps(); - processAndReportCoredumps(processingCoredumps); - } - - - /** - * Moves a coredump to a new directory under the processing/ directory. Limit to only processing - * one coredump at the time, starting with the oldest. - */ - Path enqueueCoredumps() { - Path processingCoredumpsPath = coredumpsPath.resolve(PROCESSING_DIRECTORY_NAME); - processingCoredumpsPath.toFile().mkdirs(); - if (!FileHelper.listContentsOfDirectory(processingCoredumpsPath).isEmpty()) return processingCoredumpsPath; - - FileHelper.listContentsOfDirectory(coredumpsPath).stream() - .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith(".")) - .min((Comparator.comparingLong(o -> o.toFile().lastModified()))) - .ifPresent(coredumpPath -> { - try { - enqueueCoredumpForProcessing(coredumpPath, processingCoredumpsPath); - } catch (Throwable e) { - logger.log(Level.WARNING, "Failed to process coredump " + coredumpPath, e); - } - }); - - return processingCoredumpsPath; - } - - void processAndReportCoredumps(Path processingCoredumpsPath) { - doneCoredumpsPath.toFile().mkdirs(); - - FileHelper.listContentsOfDirectory(processingCoredumpsPath).stream() - .filter(path -> path.toFile().isDirectory()) - .forEach(coredumpDirectory -> { - try { - String metadata = collectMetadata(coredumpDirectory, nodeAttributes); - report(coredumpDirectory, metadata); - finishProcessing(coredumpDirectory); - } catch (Throwable e) { - logger.log(Level.WARNING, "Failed to report coredump " + coredumpDirectory, e); - } - }); - } - - Path enqueueCoredumpForProcessing(Path coredumpPath, Path processingCoredumpsPath) throws IOException { - // Make coredump readable - coredumpPath.toFile().setReadable(true, false); - - // Create new directory for this coredump and move it into it - Path folder = processingCoredumpsPath.resolve(UUID.randomUUID().toString()); - folder.toFile().mkdirs(); - return Files.move(coredumpPath, folder.resolve(coredumpPath.getFileName())); - } - - String collectMetadata(Path coredumpDirectory, Map<String, Object> nodeAttributes) throws IOException { - Path metadataPath = coredumpDirectory.resolve(METADATA_FILE_NAME); - if (!Files.exists(metadataPath)) { - Path coredumpPath = FileHelper.listContentsOfDirectory(coredumpDirectory).stream().findFirst() - .orElseThrow(() -> new RuntimeException("No coredump file found in processing directory " + coredumpDirectory)); - Map<String, Object> metadata = coreCollector.collect(coredumpPath, installStatePath); - metadata.putAll(nodeAttributes); - - Map<String, Object> fields = new HashMap<>(); - fields.put("fields", metadata); - - String metadataFields = objectMapper.writeValueAsString(fields); - Files.write(metadataPath, metadataFields.getBytes()); - return metadataFields; - } else { - return new String(Files.readAllBytes(metadataPath)); - } - } - - void report(Path coredumpDirectory, String metadata) throws IOException { - // Use core dump UUID as document ID - String documentId = coredumpDirectory.getFileName().toString(); - - HttpPost post = new HttpPost(feedEndpoint + "/" + documentId); - post.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); - post.setEntity(new StringEntity(metadata)); - - try (CloseableHttpResponse response = httpClient.execute(post)) { - if (response.getStatusLine().getStatusCode() / 100 != 2) { - String result = new BufferedReader(new InputStreamReader(response.getEntity().getContent())) - .lines().collect(Collectors.joining("\n")); - throw new RuntimeException("POST to " + post.getURI() + " failed with HTTP: " + - response.getStatusLine().getStatusCode() + " [" + result + "]"); - } - EntityUtils.consume(response.getEntity()); - } - logger.info("Successfully reported coredump " + documentId); - } - - void finishProcessing(Path coredumpDirectory) throws IOException { - Files.move(coredumpDirectory, doneCoredumpsPath.resolve(coredumpDirectory.getFileName())); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java deleted file mode 100644 index 7b93e7ad98d..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author freva - */ -public class FileHelper { - private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName()); - - /** - * (Recursively) deletes files if they match all the criteria, also deletes empty directories. - * - * @param basePath Base path from where to start the search - * @param maxAge Delete files older (last modified date) than maxAge - * @param fileNameRegex Delete files where filename matches fileNameRegex - * @param recursive Delete files in sub-directories (with the same criteria) - */ - public static void deleteFiles(Path basePath, Duration maxAge, Optional<String> fileNameRegex, boolean recursive) throws IOException { - Pattern fileNamePattern = fileNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - if (recursive) { - deleteFiles(path, maxAge, fileNameRegex, true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } else if (isPatternMatchingFilename(fileNamePattern, path) && - isTimeSinceLastModifiedMoreThan(path, maxAge)) { - if (! Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes all files in target directory except the n most recent (by modified date) - * - * @param basePath Base path to delete from - * @param nMostRecentToKeep Number of most recent files to keep - */ - static void deleteFilesExceptNMostRecent(Path basePath, int nMostRecentToKeep) throws IOException { - if (nMostRecentToKeep < 1) { - throw new IllegalArgumentException("Number of files to keep must be a positive number"); - } - - List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream() - .filter(Files::isRegularFile) - .sorted(Comparator.comparing(FileHelper::getLastModifiedTime)) - .skip(nMostRecentToKeep) - .collect(Collectors.toList()); - - for (Path path : pathsInDeleteDir) { - if (!Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - - static void deleteFilesLargerThan(Path basePath, long sizeInBytes) throws IOException { - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - deleteFilesLargerThan(path, sizeInBytes); - } else { - if (Files.size(path) > sizeInBytes && !Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes directories and their contents if they match all the criteria - * - * @param basePath Base path to delete the directories from - * @param maxAge Delete directories older (last modified date) than maxAge - * @param dirNameRegex Delete directories where directory name matches dirNameRegex - */ - public static void deleteDirectories(Path basePath, Duration maxAge, Optional<String> dirNameRegex) throws IOException { - Pattern dirNamePattern = dirNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path) && isPatternMatchingFilename(dirNamePattern, path)) { - boolean mostRecentFileModifiedBeforeMaxAge = getMostRecentlyModifiedFileIn(path) - .map(mostRecentlyModified -> isTimeSinceLastModifiedMoreThan(mostRecentlyModified, maxAge)) - .orElse(true); - - if (mostRecentFileModifiedBeforeMaxAge) { - deleteFiles(path, Duration.ZERO, Optional.empty(), true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } - } - } - - /** - * Similar to rm -rf file: - * - It's not an error if file doesn't exist - * - If file is a directory, it and all content is removed - * - For symlinks: Only the symlink is removed, not what the symlink points to - */ - public static void recursiveDelete(Path basePath) throws IOException { - if (Files.isDirectory(basePath)) { - for (Path path : listContentsOfDirectory(basePath)) { - recursiveDelete(path); - } - } - - Files.deleteIfExists(basePath); - } - - public static void moveIfExists(Path from, Path to) throws IOException { - if (Files.exists(from)) { - Files.move(from, to); - } - } - - private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException { - return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime)); - } - - private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) { - Instant nowMinusDuration = Instant.now().minus(duration); - Instant lastModified = getLastModifiedTime(path).toInstant(); - - // Return true also if they are equal for test stability - // (lastModified <= nowMinusDuration) is the same as !(lastModified > nowMinusDuration) - return !lastModified.isAfter(nowMinusDuration); - } - - private static boolean isPatternMatchingFilename(Pattern pattern, Path path) { - return pattern == null || pattern.matcher(path.getFileName().toString()).find(); - } - - /** - * @return list all files in a directory, returns empty list if directory does not exist - */ - public static List<Path> listContentsOfDirectory(Path basePath) { - try (Stream<Path> directoryStream = Files.list(basePath)) { - return directoryStream.collect(Collectors.toList()); - } catch (NoSuchFileException ignored) { - return Collections.emptyList(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e); - } - } - - static FileTime getLastModifiedTime(Path path) { - try { - return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS); - } catch (IOException e) { - throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e); - } - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java deleted file mode 100644 index 3bf62e60481..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import com.yahoo.log.LogSetup; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Type; -import com.yahoo.system.ProcessExecuter; -import com.yahoo.vespa.config.SlimeUtils; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author freva - */ -public class Maintainer { - - private static final CoreCollector coreCollector = new CoreCollector(new ProcessExecuter()); - - public static void main(String[] args) { - LogSetup.initVespaLogging("node-maintainer"); - if (args.length != 1) { - throw new RuntimeException("Expected only 1 argument - a JSON list of maintainer jobs to execute"); - } - - Inspector object = SlimeUtils.jsonToSlime(args[0].getBytes()).get(); - if (object.type() != Type.ARRAY) { - throw new IllegalArgumentException("Expected a list of maintainer jobs to execute"); - } - - // Variable must be effectively final to be used in lambda expression - AtomicInteger numberOfJobsFailed = new AtomicInteger(0); - object.traverse((ArrayTraverser) (int i, Inspector item) -> { - try { - String type = getFieldOrFail(item, "type").asString(); - Inspector arguments = getFieldOrFail(item, "arguments"); - parseMaintenanceJob(type, arguments); - } catch (Exception e) { - System.err.println("Failed executing job: " + item.toString()); - e.printStackTrace(); - numberOfJobsFailed.incrementAndGet(); - } - }); - - if (numberOfJobsFailed.get() > 0) { - System.err.println(numberOfJobsFailed.get() + " of jobs has failed"); - System.exit(1); - } - } - - private static void parseMaintenanceJob(String type, Inspector arguments) { - if (arguments.type() != Type.OBJECT) { - throw new IllegalArgumentException("Expected a 'arguments' to be an object"); - } - - switch (type) { - case "delete-files": - parseDeleteFilesJob(arguments); - break; - - case "delete-directories": - parseDeleteDirectoriesJob(arguments); - break; - - case "recursive-delete": - parseRecursiveDelete(arguments); - break; - - case "move-files": - parseMoveFiles(arguments); - break; - - case "handle-core-dumps": - parseHandleCoreDumps(arguments); - break; - - default: - throw new IllegalArgumentException("Unknown job: " + type); - } - } - - private static void parseDeleteFilesJob(Inspector arguments) { - Path basePath = Paths.get(getFieldOrFail(arguments, "basePath").asString()); - Duration maxAge = Duration.ofSeconds(getFieldOrFail(arguments, "maxAgeSeconds").asLong()); - Optional<String> fileNameRegex = SlimeUtils.optionalString(arguments.field("fileNameRegex")); - boolean recursive = getFieldOrFail(arguments, "recursive").asBool(); - try { - FileHelper.deleteFiles(basePath, maxAge, fileNameRegex, recursive); - } catch (IOException e) { - throw new RuntimeException("Failed deleting files under " + basePath.toAbsolutePath() + - fileNameRegex.map(regex -> ", matching '" + regex + "'").orElse("") + - ", " + (recursive ? "" : "not ") + "recursively" + - " and older than " + maxAge, e); - } - } - - private static void parseDeleteDirectoriesJob(Inspector arguments) { - Path basePath = Paths.get(getFieldOrFail(arguments, "basePath").asString()); - Duration maxAge = Duration.ofSeconds(getFieldOrFail(arguments, "maxAgeSeconds").asLong()); - Optional<String> dirNameRegex = SlimeUtils.optionalString(arguments.field("dirNameRegex")); - try { - FileHelper.deleteDirectories(basePath, maxAge, dirNameRegex); - } catch (IOException e) { - throw new RuntimeException("Failed deleting directories under " + basePath.toAbsolutePath() + - dirNameRegex.map(regex -> ", matching '" + regex + "'").orElse("") + - " and older than " + maxAge, e); - } - } - - private static void parseRecursiveDelete(Inspector arguments) { - Path basePath = Paths.get(getFieldOrFail(arguments, "path").asString()); - try { - FileHelper.recursiveDelete(basePath); - } catch (IOException e) { - throw new RuntimeException("Failed deleting " + basePath.toAbsolutePath(), e); - } - } - - private static void parseMoveFiles(Inspector arguments) { - Path from = Paths.get(getFieldOrFail(arguments, "from").asString()); - Path to = Paths.get(getFieldOrFail(arguments, "to").asString()); - - try { - FileHelper.moveIfExists(from, to); - } catch (IOException e) { - throw new RuntimeException("Failed moving from " + from.toAbsolutePath() + ", to " + to.toAbsolutePath(), e); - } - } - - private static void parseHandleCoreDumps(Inspector arguments) { - Path coredumpsPath = Paths.get(getFieldOrFail(arguments, "coredumpsPath").asString()); - Path doneCoredumpsPath = Paths.get(getFieldOrFail(arguments, "doneCoredumpsPath").asString()); - Map<String, Object> attributesMap = parseMap(arguments); - Optional<Path> installStatePath = SlimeUtils.optionalString(arguments.field("installStatePath")).map(Paths::get); - String feedEndpoint = getFieldOrFail(arguments, "feedEndpoint").asString(); - - try (CloseableHttpClient httpClient = createHttpClient(Duration.ofSeconds(5))) { - CoredumpHandler coredumpHandler = new CoredumpHandler(httpClient, coreCollector, coredumpsPath, - doneCoredumpsPath, attributesMap, installStatePath, - feedEndpoint); - coredumpHandler.processAll(); - } catch (IOException e) { - throw new RuntimeException("Failed processing coredumps at " + coredumpsPath.toAbsolutePath() + - ", moving fished dumps to " + doneCoredumpsPath.toAbsolutePath(), e); - } - } - - private static Map<String, Object> parseMap(Inspector object) { - Map<String, Object> map = new HashMap<>(); - getFieldOrFail(object, "attributes").traverse((String key, Inspector value) -> { - switch (value.type()) { - case BOOL: - map.put(key, value.asBool()); - break; - case LONG: - map.put(key, value.asLong()); - break; - case DOUBLE: - map.put(key, value.asDouble()); - break; - case STRING: - map.put(key, value.asString()); - break; - default: - throw new IllegalArgumentException("Invalid attribute for key '" + key + "', value " + value); - } - }); - return map; - } - - private static Inspector getFieldOrFail(Inspector object, String key) { - Inspector out = object.field(key); - if (out.type() == Type.NIX) { - throw new IllegalArgumentException("Key '" + key + "' was not found!"); - } - return out; - } - - private static CloseableHttpClient createHttpClient(Duration timeout) { - int timeoutInMillis = (int) timeout.toMillis(); - return HttpClientBuilder.create() - .setUserAgent("node-maintainer") - .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(timeoutInMillis) - .setConnectionRequestTimeout(timeoutInMillis) - .setSocketTimeout(timeoutInMillis) - .build()) - .setMaxConnTotal(100) - .setMaxConnPerRoute(10) - .build(); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java deleted file mode 100644 index a33dd07dff2..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.log.LogLevel; -import com.yahoo.log.LogSetup; -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; -import com.yahoo.vespa.hosted.node.verification.hardware.HardwareBenchmarker; -import com.yahoo.vespa.hosted.node.verification.spec.SpecVerifier; -import io.airlift.command.Cli; -import io.airlift.command.Help; -import io.airlift.command.Option; - -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author freva - */ -public class Main { - private static final String EXECUTABLE_NAME = "node-verifier"; - private static final Logger logger = Logger.getLogger(Main.class.getName()); - private static final ObjectMapper om = new ObjectMapper(); - - public static void main(String[] args) { - LogSetup.initVespaLogging(EXECUTABLE_NAME); - - try { - System.out.println(execute(args, new CommandExecutor())); - } catch (Exception e) { - logger.log(LogLevel.ERROR, "Something went wrong", e); - System.exit(1); - } - } - - @SuppressWarnings("unchecked") - public static String execute(String[] args, CommandExecutor commandExecutor) throws IOException { - Cli.CliBuilder<Object> builder = Cli.builder(EXECUTABLE_NAME) - .withDescription("Verifies that node meets the expected specification and benchmarks") - .withDefaultCommand(Help.class) - .withCommands(Help.class, SpecVerifier.class, HardwareBenchmarker.class); - - Object command = builder.build().parse(args); - if (command instanceof VerifierCommand) { - HardwareDivergenceReport report = ((VerifierCommand) command).getPreviousHardwareDivergence(); - ((VerifierCommand) command).run(report, commandExecutor); - return hardwareDivergenceReportToString(report); - } else if (command instanceof Runnable) { - ((Runnable) command).run(); - return ""; - } - - throw new RuntimeException("Unknown command class " + command.getClass().getName()); - } - - public static abstract class VerifierCommand { - @Option(name = {"-h", "--divergence"}, description = "JSON of the previous hardware divergence report") - private String hardwareDivergence; - - private HardwareDivergenceReport getPreviousHardwareDivergence() { - if (hardwareDivergence == null) { - return new HardwareDivergenceReport(); - } - try { - return om.readValue(hardwareDivergence, HardwareDivergenceReport.class); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to parse hardware divergence:\n" + hardwareDivergence, e.getMessage()); - return new HardwareDivergenceReport(); - } - } - - protected abstract void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor); - } - - private static String hardwareDivergenceReportToString(HardwareDivergenceReport hardwareDivergenceReport) throws IOException { - if (hardwareDivergenceReport.isHardwareDivergenceReportEmpty()) { - return "null"; - } else { - return om.writeValueAsString(hardwareDivergenceReport); - } - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md deleted file mode 100644 index e7869adb66a..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md +++ /dev/null @@ -1,31 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -#TODO -Here we have listed things we did not have time to do during the summer, but that we think can be implemented later. - -##Spec -- The net interface speed is at the moment approved if it is over 1 000MB/s. Since some nodes are supposed to have -10 000MB/s a feature could be to either add information about interface speed in node repo or use the flavor to decide -if the interface speed is correct or not. -- In HardwareNodeComparator the spec found in node repo and on the node are compared. We set a threshold on 5%, meaning -if a the value from the node is more than 5% away from what is said in node repo (+-), then we say it is a bad value. -If this threshold is too high or too low, it has to be changed in this class. - -##Benchmark -- BenchmarkResultInspector is the class that decides whether a benchmark result is ok or if the result should be - reported. The values that decides this are not given very much thought and it could be an idea to check these. -- Benchmark is not running on docker hosts since there is not yet found a solution to only start benchmarking at reboot. -Spec verification runs every hour on node-admin, but since node-admin reboots too often, benchmarking is at the moment -not running here. - -##Reporting / Node repo -- Since HardwareDivergenceReport is printed as a json string and then reported to noderepo as a string, a feature could -be to not store HardwareDivergence as a string in node repo, as it is now, but as a json. This will be mostly changes -outside of this code, but then the member variable HardwareDivergence in NodeRepoJsonModel have to be change from string -to HardwareDivergenceReport which will cause some work in the Reporter class. -- No actions are now taken if there is anything wrong with the node. The information is only uploaded to node repo. -Here it is room for improvements and some possible solutions are: - - Continue only reporting to node repo, but have a program that scans through all nodes and makes a list of those - with errors in HardwareDivergence. - - In addition to uploading the report to node repo, have actions based on what kind of errors and what kind of state - the node has. - - Automatically create Jira tickets when something is reported wrong on a node. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java deleted file mode 100644 index 21e54c8c010..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons; - -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.PumpStreamHandler; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; - -/** - * Wrapper for executing terminal commands - * - * @author olaaun - * @author sgrostad - */ -public class CommandExecutor { - - public List<String> executeCommand(String command) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - List<String> results = new ArrayList<>(); - writeToOutputStream(outputStream, command); - writeOutputStreamToResults(outputStream, results); - return results; - } - - private void writeToOutputStream(ByteArrayOutputStream outputStream, String command) throws IOException { - CommandLine cmdLine = new CommandLine("/bin/bash"); - cmdLine.addArgument("-c", false); - cmdLine.addArgument(command, false); - DefaultExecutor executor = new DefaultExecutor(); - PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream); - executor.setStreamHandler(streamHandler); - executor.execute(cmdLine); - } - - private void writeOutputStreamToResults(ByteArrayOutputStream outputStream, List<String> results) throws IOException { - String out = outputStream.toString(); - BufferedReader br = new BufferedReader(new StringReader(out)); - String line; - while ((line = br.readLine()) != null) { - results.add(line); - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java deleted file mode 100644 index fe1f8d72d50..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.noderepo; - -import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; - -import javax.naming.NamingEnumeration; -import javax.naming.NamingException; -import javax.naming.directory.Attribute; -import javax.naming.directory.Attributes; -import javax.naming.directory.DirContext; -import javax.naming.directory.InitialDirContext; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.Hashtable; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Created by olaa on 14/07/2017. - * Verifies that the IP addresses of a node points to the correct hostname - * - * @author olaaun - * @author sgrostad - */ -public class IPAddressVerifier { - - private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName()); - - private final String expectedHostname; - private final boolean skipLookup; - private final boolean skipReverseLookup; - - public IPAddressVerifier(String expectedHostname, boolean skipLookup, boolean skipReverseLookup) { - this.expectedHostname = expectedHostname; - this.skipLookup = skipLookup; - this.skipReverseLookup = skipReverseLookup; - } - - public void reportFaultyIpAddresses(NodeSpec nodeSpec, SpecVerificationReport specVerificationReport) { - String[] faultyIpAddresses = getFaultyIpAddresses(nodeSpec); - if (faultyIpAddresses.length > 0) { - specVerificationReport.setFaultyIpAddresses(faultyIpAddresses); - } - } - - public String[] getFaultyIpAddresses(NodeSpec nodeSpec) { - List<String> faultyIpAddresses = new ArrayList<>(); - if (expectedHostname == null || expectedHostname.equals("")) - return new String[0]; - - if (!isValidIpv4(nodeSpec.getIpv4Address())) { - faultyIpAddresses.add(nodeSpec.getIpv4Address()); - } - if (!isValidIpv6(nodeSpec.getIpv6Address())) { - faultyIpAddresses.add(nodeSpec.getIpv6Address()); - } - return faultyIpAddresses.toArray(new String[0]); - } - - private boolean hostnameResolvesToIpAddress(String ipAddress) { - if (skipLookup) { - return true; - } - - InetAddress addressFromIpAddress; - try { - addressFromIpAddress = InetAddress.getByName(ipAddress); - } catch (UnknownHostException e) { - logger.log(Level.WARNING, "Failed to parse IP address " + ipAddress, e); - return false; - } - - List<InetAddress> addressesFromHostname; - try { - addressesFromHostname = mockableGetAllByName(expectedHostname); - } catch (UnknownHostException e) { - logger.log(Level.WARNING, "Failed to get IP addresses of hostname " + expectedHostname, e); - return false; - } - - if (addressesFromHostname.stream().noneMatch(addressFromIpAddress::equals)) { - logger.log(Level.WARNING, "Hostname " + expectedHostname + " resolved to " + addressesFromHostname + - " which does not contain the IP address " + addressFromIpAddress); - return false; - } - - return true; - } - - private boolean ipAddressResolvesToHostname(String ipAddressLookupFormat) { - if (skipReverseLookup) { - return true; - } - - try { - String hostnameFromIpAddress = reverseLookUp(ipAddressLookupFormat); - if (hostnameFromIpAddress.equals(expectedHostname)) { - return true; - } - - logger.log(Level.WARNING, "IP address " + ipAddressLookupFormat + " resolved to " + - hostnameFromIpAddress + ", not " + expectedHostname); - } catch (NamingException e) { - logger.log(Level.WARNING, "Could not get hostname of IP address " + ipAddressLookupFormat, e); - } - - return false; - } - - private boolean isValidIpv4(String ipv4Address) { - if (ipv4Address == null) { - return true; - } - - return hostnameResolvesToIpAddress(ipv4Address) && - ipAddressResolvesToHostname(convertIpv4ToLookupFormat(ipv4Address)); - } - - private boolean isValidIpv6(String ipv6Address) { - if (ipv6Address == null) { - return true; - } - - return hostnameResolvesToIpAddress(ipv6Address) && - ipAddressResolvesToHostname(convertIpv6ToLookupFormat(ipv6Address)); - } - - String reverseLookUp(String ipAddress) throws NamingException { - Hashtable<String, String> env = new Hashtable<>(); - env.put("java.naming.factory.initial", "com.sun.jndi.dns.DnsContextFactory"); - DirContext ctx = new InitialDirContext(env); - Attributes attrs = ctx.getAttributes(ipAddress, new String[]{"PTR"}); - for (NamingEnumeration<? extends Attribute> ae = attrs.getAll(); ae.hasMoreElements(); ) { - Attribute attr = ae.next(); - Enumeration<?> vals = attr.getAll(); - if (vals.hasMoreElements()) { - String hostname = vals.nextElement().toString(); - ctx.close(); - return hostname.substring(0, hostname.length() - 1); - } - } - ctx.close(); - return ""; - } - - String convertIpv6ToLookupFormat(String ipAddress) { - StringBuilder newIpAddress = new StringBuilder(); - String doubleColonReplacement = "0.0.0.0.0.0.0.0.0.0.0.0."; - String domain = "ip6.arpa"; - String[] hextets = ipAddress.split(":"); - for (int i = hextets.length - 1; i >= 0; i--) { - String reversedHextet = new StringBuilder(hextets[i]).reverse().toString(); - if (reversedHextet.equals("")) { - newIpAddress.append(doubleColonReplacement); - continue; - } - String trailingZeroes = "0000"; - String paddedHextet = (reversedHextet + trailingZeroes).substring(0, trailingZeroes.length()); - String punctuatedHextet = paddedHextet.replaceAll(".", "$0."); - newIpAddress.append(punctuatedHextet); - } - newIpAddress.append(domain); - return newIpAddress.toString(); - } - - String convertIpv4ToLookupFormat(String ipAddress) { - String domain = "in-addr.arpa"; - String[] octets = ipAddress.split("\\."); - StringBuilder convertedIpAddress = new StringBuilder(); - for (int i = octets.length - 1; i >= 0; i--) { - convertedIpAddress.append(octets[i]).append("."); - } - convertedIpAddress.append(domain); - return convertedIpAddress.toString(); - } - - List<InetAddress> mockableGetAllByName(String hostname) throws UnknownHostException { - return Arrays.asList(InetAddress.getAllByName(hostname)); - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java deleted file mode 100644 index 622d46ca587..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.noderepo; - -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; - -/** - * Converts a NodeSpec object to a HardwareInfo object. - * - * @author olaaun - * @author sgrostad - */ -public class NodeJsonConverter { - - private static void setInterfaceSpeed(NodeSpec nodeSpec, HardwareInfo nodeRepoHardwareInfo) { - nodeRepoHardwareInfo.setInterfaceSpeedMbs(nodeSpec.getBandwidth()); - } - - private static void setIpv6Interface(NodeSpec nodeSpec, HardwareInfo nodeRepoHardwareInfo) { - if (nodeSpec.getIpv6Address() != null) { - nodeRepoHardwareInfo.setIpv6Interface(true); - } - } - - private static void setIpv4Interface(NodeSpec nodeSpec, HardwareInfo nodeRepoHardwareInfo) { - if (nodeSpec.getIpv4Address() != null) { - nodeRepoHardwareInfo.setIpv4Interface(true); - } - } - - public static HardwareInfo convertJsonModelToHardwareInfo(NodeSpec nodeSpec) { - HardwareInfo nodeRepoHardwareInfo = nodeSpec.copyToHardwareInfo(); - setInterfaceSpeed(nodeSpec, nodeRepoHardwareInfo); - setIpv4Interface(nodeSpec, nodeRepoHardwareInfo); - setIpv6Interface(nodeSpec, nodeRepoHardwareInfo); - return nodeRepoHardwareInfo; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java deleted file mode 100644 index 16be1c39a74..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.noderepo; - -import com.google.common.net.InetAddresses; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; - -import java.net.Inet4Address; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.util.stream.Stream; - -/** - * Object with the information about a node - * - * @author freva - */ -public class NodeSpec { - private final double minDiskAvailableGb; - private final double minMainMemoryAvailableGb; - private final double minCpuCores; - private final boolean fastDisk; - private final double bandwidth; - private final String[] ipAddresses; - - public NodeSpec(double minDiskAvailableGb, double minMainMemoryAvailableGb, double minCpuCores, boolean fastDisk, - double bandwidth, String[] ipAddresses) { - this.minDiskAvailableGb = minDiskAvailableGb; - this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; - this.minCpuCores = minCpuCores; - this.fastDisk = fastDisk; - this.bandwidth = bandwidth; - this.ipAddresses = ipAddresses; - } - - public HardwareInfo copyToHardwareInfo() { - HardwareInfo hardwareInfo = new HardwareInfo(); - hardwareInfo.setMinMainMemoryAvailableGb(this.minMainMemoryAvailableGb); - hardwareInfo.setMinDiskAvailableGb(this.minDiskAvailableGb); - hardwareInfo.setMinCpuCores((int) Math.round(this.minCpuCores)); - hardwareInfo.setDiskType(this.fastDisk ? DiskType.FAST : DiskType.SLOW); - hardwareInfo.setInterfaceSpeedMbs(bandwidth); - hardwareInfo.setIpv6Connection(getIpv6Address() != null); - return hardwareInfo; - } - - public String getIpv6Address() { - return Stream.of(ipAddresses) - .map(InetAddresses::forString) - .filter(ip -> ip instanceof Inet6Address) - .findFirst().map(InetAddress::getHostAddress).orElse(null); - } - - public String getIpv4Address() { - return Stream.of(ipAddresses) - .map(InetAddresses::forString) - .filter(ip -> ip instanceof Inet4Address) - .findFirst().map(InetAddress::getHostAddress).orElse(null); - } - - public double getBandwidth() { return bandwidth ; } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java deleted file mode 100644 index 55bf1dd0e0b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.parser; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.regex.Pattern; - -/** - * Parses terminal command output, and returns results based on ParseInstructions - * - * @author sgrostad - * @author olaaaun - */ -public class OutputParser { - - public static List<ParseResult> parseOutput(ParseInstructions parseInstructions, List<String> commandOutput) { - List<ParseResult> results = new ArrayList<>(); - int searchElementIndex = parseInstructions.getSearchElementIndex(); - int valueElementIndex = parseInstructions.getValueElementIndex(); - List<String> searchWords = parseInstructions.getSearchWords(); - for (String line : commandOutput) { - String[] lineSplit = line.trim().split(parseInstructions.getSplitRegex()); - if (lineSplit.length <= Math.max(searchElementIndex, valueElementIndex)) { - continue; - } - String searchWordCandidate = lineSplit[searchElementIndex].trim(); - boolean searchWordCandidateMatch = matchingSearchWord(searchWords, searchWordCandidate); - if (searchWordCandidateMatch) { - String value = lineSplit[valueElementIndex]; - results.add(new ParseResult(searchWordCandidate, value.trim())); - } - } - return results; - } - - public static Optional<ParseResult> parseSingleOutput(ParseInstructions parseInstructions, List<String> commandOutput) { - List<ParseResult> parseResults = parseOutput(parseInstructions, commandOutput); - if (parseResults.isEmpty()) return Optional.empty(); - return Optional.ofNullable(parseResults.get(0)); - } - - private static boolean matchingSearchWord(List<String> searchWords, String searchWordCandidate) { - return searchWords.stream().anyMatch(w -> Pattern.compile(w).matcher(searchWordCandidate).matches()); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java deleted file mode 100644 index 262bfcba0b9..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.parser; - -import java.util.List; - -/** - * Contains instructions of how a command line output should be parsed - * - * @author sgrostad - * @author olaaaun - */ -public class ParseInstructions { - - private final int searchElementIndex; - private final int valueElementIndex; - private final String splitRegex; - private final List<String> searchWords; - - public ParseInstructions(int searchElementIndex, int returnElementNum, String splitRegex, List<String> searchWords) { - this.searchElementIndex = searchElementIndex; - this.valueElementIndex = returnElementNum; - this.splitRegex = splitRegex; - this.searchWords = searchWords; - } - - public int getSearchElementIndex() { - return searchElementIndex; - } - - public int getValueElementIndex() { - return valueElementIndex; - } - - public String getSplitRegex() { - return splitRegex; - } - - public List<String> getSearchWords() { - return searchWords; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java deleted file mode 100644 index a89cfd89f31..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.parser; - -import java.util.Objects; - -/** - * Contains the results from parsing a command line output - * - * @author sgrostad - * @author olaaun - */ -public class ParseResult { - - private final String searchWord; - private final String value; - - public ParseResult(String searchWord, String value) { - this.searchWord = searchWord; - this.value = value; - } - - public String getSearchWord() { - return searchWord; - } - - public String getValue() { - return value; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj instanceof ParseResult) { - ParseResult parseResult = (ParseResult) obj; - if (this.searchWord.equals(parseResult.getSearchWord()) && this.value.equals(parseResult.getValue())) { - return true; - } - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(searchWord, value); - } - - @Override - public String toString() { - return "Search word: " + searchWord + ", Value: " + value; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java deleted file mode 100644 index ce4e3685a25..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.report; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -/** - * JSON-mapped class for reporting benchmark results to node repo - * - * @author sgrostad - * @author olaaun - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class BenchmarkReport { - - @JsonProperty - private Double cpuCyclesPerSec; - @JsonProperty - private Double diskSpeedMbs; - @JsonProperty - private Double memoryWriteSpeedGBs; - @JsonProperty - private Double memoryReadSpeedGBs; - - public void setCpuCyclesPerSec(double cpuCyclesPerSec) { - this.cpuCyclesPerSec = cpuCyclesPerSec; - } - - public void setDiskSpeedMbs(Double diskSpeedMbs) { - this.diskSpeedMbs = diskSpeedMbs != null ? diskSpeedMbs : -1; - } - - - public void setMemoryWriteSpeedGBs(Double memoryWriteSpeedGBs) { - this.memoryWriteSpeedGBs = memoryWriteSpeedGBs != null ? memoryWriteSpeedGBs : -1; - } - - public void setMemoryReadSpeedGBs(Double memoryReadSpeedGBs) { - this.memoryReadSpeedGBs = memoryReadSpeedGBs != null ? memoryReadSpeedGBs : -1; - } - - public Double getCpuCyclesPerSec() { - return cpuCyclesPerSec; - } - - public Double getDiskSpeedMbs() { - return diskSpeedMbs; - } - - public Double getMemoryWriteSpeedGBs() { - return memoryWriteSpeedGBs; - } - - public Double getMemoryReadSpeedGBs() { - return memoryReadSpeedGBs; - } - - @JsonIgnore - public boolean isAllBenchmarksOK() { - ObjectMapper om = new ObjectMapper(); - try { - String jsonReport = om.writeValueAsString(this); - return jsonReport.length() == 2; - } catch (JsonProcessingException e) { - e.printStackTrace(); - return false; - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java deleted file mode 100644 index 972733abdd2..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.report; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON-wrapped report for node repo - * - * @author sgrostad - * @author olaaun - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class HardwareDivergenceReport { - - @JsonProperty - SpecVerificationReport specVerificationReport; - - @JsonProperty - BenchmarkReport benchmarkReport; - - public void setSpecVerificationReport(SpecVerificationReport specVerificationReport) { - if (specVerificationReport.isValidSpec()) { - this.specVerificationReport = null; - } else { - this.specVerificationReport = specVerificationReport; - } - } - - public void setBenchmarkReport(BenchmarkReport benchmarkReport) { - if (benchmarkReport.isAllBenchmarksOK()) { - this.benchmarkReport = null; - } else { - this.benchmarkReport = benchmarkReport; - } - } - - @JsonIgnore - public boolean isHardwareDivergenceReportEmpty() { - return specVerificationReport == null && benchmarkReport == null; - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java deleted file mode 100644 index 0494eed5dc5..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.report; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; - - -/** - * Stores results of comparing node repo spec and actual hardware info. - * In case of divergent values, set the corresponding attribute to the actual hardware info value. - * Attributes of equal value remain null. - * - * @author sgrostad - * @author olaaun - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class SpecVerificationReport { - - @JsonProperty - private Double actualMemoryAvailable; - @JsonProperty - private HardwareInfo.DiskType actualDiskType; - @JsonProperty - private Double actualDiskSpaceAvailable; - @JsonProperty - private Double actualInterfaceSpeed; - @JsonProperty - private Integer actualcpuCores; - @JsonProperty - private String[] faultyIpAddresses; - @JsonProperty - private Boolean actualIpv6Connection; - - public void setActualIpv6Connection(boolean actualIpv6Connection) { - this.actualIpv6Connection = actualIpv6Connection; - } - - public void setActualMemoryAvailable(Double actualMemoryAvailable) { - this.actualMemoryAvailable = actualMemoryAvailable; - } - - public void setActualDiskType(HardwareInfo.DiskType actualFastDisk) { - this.actualDiskType = actualFastDisk; - } - - public void setActualDiskSpaceAvailable(Double actualDiskSpaceAvailable) { - this.actualDiskSpaceAvailable = actualDiskSpaceAvailable; - } - - public void setActualcpuCores(int actualcpuCores) { - this.actualcpuCores = actualcpuCores; - } - - public void setActualInterfaceSpeed(Double actualInterfaceSpeed) { - this.actualInterfaceSpeed = actualInterfaceSpeed; - } - - public void setFaultyIpAddresses(String[] faultyIpAddresses) { - this.faultyIpAddresses = faultyIpAddresses; - } - - @JsonIgnore - public boolean isValidSpec() { - ObjectMapper om = new ObjectMapper(); - try { - String jsonReport = om.writeValueAsString(this); - return jsonReport.length() == 2; - } catch (JsonProcessingException e) { - e.printStackTrace(); - return false; - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java deleted file mode 100644 index 5e63706f556..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; - -/** - * Responsible for checking the benchmarks results, and adding unreasonable results to BenchmarkReport - * - * @author sgrostad - * @author olaaun - */ -public class BenchmarkResultInspector { - - private static final double CPU_FREQUENCY_LOWER_LIMIT = 0.5; - private static final double MEMORY_WRITE_SPEED_LOWER_LIMIT = 1D; - private static final double MEMORY_READ_SPEED_LOWER_LIMIT = 1D; - private static final double DISK_SPEED_LOWER_LIMIT = 50D; - - public static BenchmarkReport makeBenchmarkReport(BenchmarkResults benchmarkResults) { - BenchmarkReport benchmarkReport = new BenchmarkReport(); - double cpuCyclesPerSec = benchmarkResults.getCpuCyclesPerSec(); - if (cpuCyclesPerSec < CPU_FREQUENCY_LOWER_LIMIT) { - benchmarkReport.setCpuCyclesPerSec(cpuCyclesPerSec); - } - double memoryWriteSpeed = benchmarkResults.getMemoryWriteSpeedGBs(); -// TODO: Temporarily disabled due to Meltdown/Spectre performance impact, see VESPA-11051 -// if (memoryWriteSpeed < MEMORY_WRITE_SPEED_LOWER_LIMIT) { -// benchmarkReport.setMemoryWriteSpeedGBs(memoryWriteSpeed); -// } -// double memoryReadSpeed = benchmarkResults.getMemoryReadSpeedGBs(); -// if (memoryReadSpeed < MEMORY_READ_SPEED_LOWER_LIMIT) { -// benchmarkReport.setMemoryReadSpeedGBs(memoryReadSpeed); -// } - double diskSpeed = benchmarkResults.getDiskSpeedMbs(); - if (diskSpeed < DISK_SPEED_LOWER_LIMIT) { - benchmarkReport.setDiskSpeedMbs(diskSpeed); - } - return benchmarkReport; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java deleted file mode 100644 index 9b3d8f45e1d..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.yahoo.vespa.hosted.node.verification.Main; -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; -import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark; -import io.airlift.command.Command; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Benchmarks different hardware components and creates report - */ -@Command(name = "benchmark", description = "Run node benchmarks") -public class HardwareBenchmarker extends Main.VerifierCommand { - - @Override - public void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor) { - BenchmarkReport benchmarkReport = hardwareBenchmarks(commandExecutor); - - hardwareDivergenceReport.setBenchmarkReport(benchmarkReport); - } - - private BenchmarkReport hardwareBenchmarks(CommandExecutor commandExecutor) { - BenchmarkResults benchmarkResults = new BenchmarkResults(); - List<Benchmark> benchmarks = new ArrayList<>(Arrays.asList( - new DiskBenchmark(benchmarkResults, commandExecutor), - new CPUBenchmark(benchmarkResults, commandExecutor), - new MemoryBenchmark(benchmarkResults, commandExecutor))); - for (Benchmark benchmark : benchmarks) { - benchmark.doBenchmark(); - } - return BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md deleted file mode 100644 index defa24bfcab..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md +++ /dev/null @@ -1,14 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Hardware Verification -Verification of behaviour and performance of hardware. Benchmarks cpu frequency, disk write speed and memory write/read speed. -A report is sent to the node repository if any of the results are below an accepted threshold. - -## Code Walkthrough -The main class, HardwareBenchmarker, calls every benchmark in the benchmark package. - -The results of these benchmarks are passed through -the BenchmarkResultInspector, which creates a BenchmarkReport containing the values below the accepted threshold. - -ReportSender is then called such that -the old HardwareDivergence report is retrieved from node repo and updated with the new results. ReportSender prints the new HardwareReport such that it -can be updated in node repo by chef or other. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java deleted file mode 100644 index 1e5b512f492..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -/** - * @author sgrostad - * @author olaaun - */ -public interface Benchmark { - - /** - * Should perform benchmark for some part of the hardware, and store the result in BenchmarkResults instance passed to class - */ - void doBenchmark(); - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java deleted file mode 100644 index dfa436460e1..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -/** - * Stores results from benchmarks - * - * @author sgrostad - * @author olaaun - */ -// TODO: This should be immutable -public class BenchmarkResults { - - private double cpuCyclesPerSec; - private double diskSpeedMbs; - private double memoryWriteSpeedGBs; - private double memoryReadSpeedGBs; - - public double getMemoryWriteSpeedGBs() { - return memoryWriteSpeedGBs; - } - - public void setMemoryWriteSpeedGBs(double memoryWriteSpeedGBs) { - this.memoryWriteSpeedGBs = memoryWriteSpeedGBs; - } - - public double getMemoryReadSpeedGBs() { - return memoryReadSpeedGBs; - } - - public void setMemoryReadSpeedGBs(double memoryReadSpeedGBs) { - this.memoryReadSpeedGBs = memoryReadSpeedGBs; - } - - public double getCpuCyclesPerSec() { - return cpuCyclesPerSec; - } - - public void setCpuCyclesPerSec(double cpuCycles) { - this.cpuCyclesPerSec = cpuCycles; - } - - public double getDiskSpeedMbs() { - return diskSpeedMbs; - } - - public void setDiskSpeedMbs(double diskSpeedMbs) { - this.diskSpeedMbs = diskSpeedMbs; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java deleted file mode 100644 index 57763177a5b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Responsible for benchmarking CPU clock frequency, and storing the result in a BenchmarkResults instance - * - * @author sgrostad - * @author olaaun - */ -public class CPUBenchmark implements Benchmark { - - private static final String CPU_BENCHMARK_COMMAND = "perf stat -e cycles dd if=/dev/zero of=/dev/null count=2000000 2>&1 | grep 'cycles\\|seconds'"; - private static final String CYCLES_SEARCH_WORD = "cycles"; - private static final String SECONDS_SEARCH_WORD = "seconds"; - private static final String SPLIT_REGEX_STRING = "\\s+"; - private static final int SEARCH_ELEMENT_INDEX = 1; - private static final int RETURN_ELEMENT_INDEX = 0; - private static final Logger logger = Logger.getLogger(CPUBenchmark.class.getName()); - private final BenchmarkResults benchmarkResults; - - private final CommandExecutor commandExecutor; - - public CPUBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { - this.benchmarkResults = benchmarkResults; - this.commandExecutor = commandExecutor; - } - - @Override - public void doBenchmark() { - try { - List<String> commandOutput = commandExecutor.executeCommand(CPU_BENCHMARK_COMMAND); - List<ParseResult> parseResults = parseCpuCyclesPerSec(commandOutput); - setCpuCyclesPerSec(parseResults); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to perform CPU benchmark", e); - } - } - - List<ParseResult> parseCpuCyclesPerSec(List<String> commandOutput) { - List<String> searchWords = new ArrayList<>(Arrays.asList(CYCLES_SEARCH_WORD, SECONDS_SEARCH_WORD)); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); - return OutputParser.parseOutput(parseInstructions, commandOutput); - } - - - void setCpuCyclesPerSec(List<ParseResult> parseResults) { - double cpuCyclesPerSec = getCyclesPerSecond(parseResults); - if (cpuCyclesPerSec > 0) { - benchmarkResults.setCpuCyclesPerSec(cpuCyclesPerSec); - } - } - - private double getCyclesPerSecond(List<ParseResult> parseResults) { - double cycles = -1; - double seconds = -1; - for (ParseResult parseResult : parseResults) { - switch (parseResult.getSearchWord()) { - case CYCLES_SEARCH_WORD: - cycles = makeCyclesDouble(parseResult.getValue()); - break; - case SECONDS_SEARCH_WORD: - seconds = makeSecondsDouble(parseResult.getValue()); - break; - default: - throw new RuntimeException("Invalid ParseResult searchWord: " + parseResult.getSearchWord()); - } - } - if (cycles > 0 && seconds > 0) { - return convertToGHz(cycles, seconds); - } - return -1; - } - - double makeCyclesDouble(String cycles) { - cycles = cycles.replaceAll("[^\\d]", ""); - if (checkIfNumber(cycles)) { - return Double.parseDouble(cycles); - } - return -1; - } - - double makeSecondsDouble(String seconds) { - seconds = seconds.replaceAll(",", "."); - if (checkIfNumber(seconds)) { - return Double.parseDouble(seconds); - } - return -1; - } - - boolean checkIfNumber(String numberCandidate) { - if (numberCandidate == null || numberCandidate.equals("")) { - return false; - } - try { - Double.parseDouble(numberCandidate); - } catch (NumberFormatException e) { - return false; - } - return true; - } - - double convertToGHz(double cycles, double seconds) { - double giga = 1000000000.0; - return (cycles / seconds) / giga; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java deleted file mode 100644 index a7fc5809058..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - - -/** - * Responsible for benchmarking disk write speed, and storing the result in a BenchmarkResults instance - * - * @author olaaun - * @author sgrostad - */ -public class DiskBenchmark implements Benchmark { - - private static final String DISK_BENCHMARK_COMMAND = "(dd if=/dev/zero of=/home/y/tmp/output conv=fdatasync bs=4G count=4; rm -f /home/y/tmp/output;) 2>&1 | grep bytes | awk '{ print $8 \" \" $9 }'"; - private static final String KILO_BYTE_SEARCH_WORD = "kB/s"; - private static final String MEGA_BYTE_SEARCH_WORD = "MB/s"; - private static final String GIGA_BYTE_SEARCH_WORD = "GB/s"; - private static final String SPLIT_REGEX_STRING = " "; - private static final int SEARCH_ELEMENT_INDEX = 1; - private static final int RETURN_ELEMENT_INDEX = 0; - private static final Logger logger = Logger.getLogger(DiskBenchmark.class.getName()); - private final BenchmarkResults benchmarkResults; - private final CommandExecutor commandExecutor; - - public DiskBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { - this.benchmarkResults = benchmarkResults; - this.commandExecutor = commandExecutor; - } - - @Override - public void doBenchmark() { - try { - List<String> commandOutput = commandExecutor.executeCommand(DISK_BENCHMARK_COMMAND); - Optional<ParseResult> parseResult = parseDiskSpeed(commandOutput); - setDiskSpeed(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to perform disk benchmark", e); - } - } - - Optional<ParseResult> parseDiskSpeed(List<String> commandOutput) { - List<String> searchWords = new ArrayList<>(Arrays.asList(KILO_BYTE_SEARCH_WORD, MEGA_BYTE_SEARCH_WORD, GIGA_BYTE_SEARCH_WORD)); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput); - } - - void setDiskSpeed(Optional<ParseResult> parseResult) { - benchmarkResults.setDiskSpeedMbs(parseResult.map(this::getDiskSpeedInMBs).orElse(0.0d)); - } - - double getDiskSpeedInMBs(ParseResult parseResult) { - double diskSpeedMBs = 0; - double convertKBsToMBs = 1 / 1000.0; - double convertGBsToMBs = 1000.0; - double convertMbsToMBs = 1.0; - String diskSpeed = parseResult.getValue(); - if (checkSpeedValidity(diskSpeed)) { - switch (parseResult.getSearchWord()) { - case KILO_BYTE_SEARCH_WORD: - diskSpeedMBs = convertToMBs(diskSpeed, convertKBsToMBs); - break; - case MEGA_BYTE_SEARCH_WORD: - diskSpeedMBs = convertToMBs(diskSpeed, convertMbsToMBs); - break; - case GIGA_BYTE_SEARCH_WORD: - diskSpeedMBs = convertToMBs(diskSpeed, convertGBsToMBs); - break; - default: - throw new RuntimeException("Invalid ParseResult searchWord: " + parseResult.getSearchWord()); - } - } - - return diskSpeedMBs; - } - - boolean checkSpeedValidity(String speed) { - try { - Double.parseDouble(speed); - } catch (NullPointerException | NumberFormatException e) { - return false; - } - return true; - } - - Double convertToMBs(String speed, double numberToConvert) { - double speedMbs = Double.parseDouble(speed); - return speedMbs * numberToConvert; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java deleted file mode 100644 index 7a72999a1e4..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Responsible for benchmarking memory read/write speed, and storing the result in a BenchmarkResults instance - * - * @author sgrostad - * @author olaaun - */ -public class MemoryBenchmark implements Benchmark { - - private static final String MEM_BENCHMARK_CREATE_FOLDER = "mkdir -p RAM_test"; - private static final String MEM_BENCHMARK_MOUNT_TMPFS = "mount tmpfs -t tmpfs RAM_test/"; - private static final String MEM_BENCHMARK_UNMOUNT_TMPFS = "umount RAM_test"; - private static final String MEM_BENCHMARK_DELETE_FOLDER = "rm -rf RAM_test"; - private static final String MEM_BENCHMARK_WRITE_SPEED = "dd if=/dev/zero of=RAM_test/data_tmp bs=1M count=512"; - private static final String MEM_BENCHMARK_READ_SPEED = "dd if=RAM_test/data_tmp of=/dev/null bs=1M count=512"; - private static final String READ_AND_WRITE_SEARCH_WORD_GB = "GB/s"; - private static final String READ_AND_WRITE_SEARCH_WORD_MB = "MB/s"; - private static final String SPLIT_REGEX_STRING = " "; - private static final int SEARCH_ELEMENT_INDEX = 8; - private static final int RETURN_ELEMENT_INDEX = 7; - private static final Logger logger = Logger.getLogger(MemoryBenchmark.class.getName()); - private final BenchmarkResults benchmarkResults; - private final CommandExecutor commandExecutor; - - public MemoryBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { - this.benchmarkResults = benchmarkResults; - this.commandExecutor = commandExecutor; - } - - @Override - public void doBenchmark() { - try { - setupMountPoint(); - - parseMemorySpeed(commandExecutor.executeCommand(MEM_BENCHMARK_WRITE_SPEED)) - .ifPresent(benchmarkResults::setMemoryWriteSpeedGBs); - - parseMemorySpeed(commandExecutor.executeCommand(MEM_BENCHMARK_READ_SPEED)) - .ifPresent(benchmarkResults::setMemoryReadSpeedGBs); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to perform memory benchmark", e); - } finally { - breakDownMountPoint(); - } - } - - private void setupMountPoint() throws IOException { - commandExecutor.executeCommand(MEM_BENCHMARK_CREATE_FOLDER); - commandExecutor.executeCommand(MEM_BENCHMARK_MOUNT_TMPFS); - } - - private void breakDownMountPoint() { - try { - commandExecutor.executeCommand(MEM_BENCHMARK_UNMOUNT_TMPFS); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to unmount tmpfs folder", e); - } - try { - commandExecutor.executeCommand(MEM_BENCHMARK_DELETE_FOLDER); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to delete memory benchmark folder", e); - } - } - - protected Optional<Double> parseMemorySpeed(List<String> commandOutput) { - Optional<ParseResult> parseResultGb = parseMemorySpeed(commandOutput, READ_AND_WRITE_SEARCH_WORD_GB); - if (parseResultGb.isPresent()) return parseDouble(parseResultGb.get().getValue()); - - Optional<ParseResult> parseResultMb = parseMemorySpeed(commandOutput, READ_AND_WRITE_SEARCH_WORD_MB); - if (parseResultMb.isPresent()) return parseDouble(parseResultMb.get().getValue()).map(v -> v / 1000.0d); - - return Optional.empty(); - } - - private Optional<ParseResult> parseMemorySpeed(List<String> commandOutput, String searchWord) { - List<String> searchWords = Collections.singletonList(searchWord); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput); - } - - private Optional<Double> parseDouble(String benchmarkOutput) { - try { - return Optional.of(Double.parseDouble(benchmarkOutput)); - } catch (NumberFormatException | NullPointerException e) { - return Optional.empty(); - } - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java deleted file mode 100644 index 57d83694709..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; - -/** - * Compares two HardwareInfo objects and stores divergent values in a SpecVerificationReport - * - * @author olaaun - * @author sgrostad - */ -public class HardwareNodeComparator { - - private static final double PERCENTAGE_THRESHOLD = 0.05; - - public static SpecVerificationReport compare(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware) { - SpecVerificationReport specVerificationReport = new SpecVerificationReport(); - if (nodeRepoHardwareInfo == null || actualHardware == null) { - return specVerificationReport; - } - setReportMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - return specVerificationReport; - } - - private static void setReportMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - setMemoryMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - setCpuMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - setDiskTypeMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - setDiskSpaceMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - setNetMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - } - - private static void setMemoryMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - double expectedMemory = nodeRepoHardwareInfo.getMinMainMemoryAvailableGb(); - double actualMemory = actualHardware.getMinMainMemoryAvailableGb(); - if (belowThreshold(expectedMemory, actualMemory, PERCENTAGE_THRESHOLD)) { - specVerificationReport.setActualMemoryAvailable(actualMemory); - } - } - - private static void setCpuMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - int expectedCpuCores = nodeRepoHardwareInfo.getMinCpuCores(); - int actualCpuCores = actualHardware.getMinCpuCores(); - if (expectedCpuCores != actualCpuCores) { - specVerificationReport.setActualcpuCores(actualCpuCores); - } - } - - private static void setDiskTypeMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - DiskType expectedFastDisk = nodeRepoHardwareInfo.getDiskType(); - DiskType actualFastDisk = actualHardware.getDiskType(); - if (expectedFastDisk != null && actualFastDisk != null && expectedFastDisk != actualFastDisk) { - specVerificationReport.setActualDiskType(actualFastDisk); - } - } - - private static void setDiskSpaceMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - double expectedDiskSpace = nodeRepoHardwareInfo.getMinDiskAvailableGb(); - double actualDiskSpace = actualHardware.getMinDiskAvailableGb(); - if (belowThreshold(expectedDiskSpace, actualDiskSpace, PERCENTAGE_THRESHOLD)) { - specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpace); - } - } - - private static void setNetMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - double expectedInterfaceSpeed = nodeRepoHardwareInfo.getInterfaceSpeedMbs(); - double actualInterfaceSpeed = actualHardware.getInterfaceSpeedMbs(); - if (expectedInterfaceSpeed > actualInterfaceSpeed) { - specVerificationReport.setActualInterfaceSpeed(actualInterfaceSpeed); - } - - if (nodeRepoHardwareInfo.isIpv6Connection() && !actualHardware.isIpv6Connection()) { - specVerificationReport.setActualIpv6Connection(actualHardware.isIpv6Connection()); - } - } - - private static boolean belowThreshold(double expected, double actual, double thresholdPercentage) { - double lowerThresholdPercentage = 1 - thresholdPercentage; - return actual < expected * lowerThresholdPercentage; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md deleted file mode 100644 index d7129553416..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md +++ /dev/null @@ -1,15 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Spec Verification -Verifies that the spec information in node repo coincides with what found on the node and reports back to node repo. - -## Code "walkthrough" -The main class SpecVerifier uses the "noderepo" package in "commons" to retrieve node spec from node repo. - -It must be called with one parameter, the config server hostname. It finds hostname using "HostURLGenerator" in the "commons" package. - -It then retrieves all the hardware information at the node with the "retrievers" package and stores the values as a -"HardwareInfo" object. - -SpecVerifier then uses HardwareNodeComparator to compare spec from node repo and the node itself. It generates a -SpecVerificationReport and uses Reporter in "commons" to retrieve the old HardwareDivergence report from node repo and update it. Reporter then prints the new -updated HardwareDivergence report such that it can be updated in node repo by chef or other. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java deleted file mode 100644 index 3f0391193ca..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.google.common.base.Strings; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.node.verification.Main; -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.IPAddressVerifier; -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeJsonConverter; -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeSpec; -import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; -import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetriever; -import io.airlift.command.Command; -import io.airlift.command.Option; - -import java.util.Optional; - -/** - * Creates two HardwareInfo objects, one with spec from node repository and one from spec retrieved at the node. - * Compares the objects and returns the result. - * - * @author olaaun - * @author sgrostad - */ -@Command(name = "specification", description = "Verify that node's actual hardware and configuration matches the expected") -public class SpecVerifier extends Main.VerifierCommand { - - @Option(name = {"-d", "--disk"}, required = true, description = "Expected disk size in GB") - private double diskAvailableGb; - - @Option(name = {"-m", "--memory"}, required = true, description = "Expected main memory size in GB") - private double mainMemoryAvailableGb; - - @Option(name = {"-c", "--cpu_cores"}, required = true, description = "Expected number of CPU cores") - private double cpuCores; - - @Option(name = {"-s", "--is_ssd"}, required = true, description = "Set to true if disk is SSD", allowedValues = {"true", "false"}) - private String fastDisk; - - @Option(name = {"-b", "--bandwidth"}, required = true, description = "Expected network interface speed in Mbit/s") - private double bandwidth; - - @Option(name = {"-i", "--ips"}, description = "Comma separated list of IP addresses assigned to this node") - private String ipAddresses; - - @Option(name = {"--skip-lookup"}, required = false, description = "Skip verification of hostname -> IP addresses") - private boolean skipLookup = false; - - @Option(name = {"--skip-reverse-lookup"}, required = false, description = "Skip verification of IP addresses -> hostname") - private boolean skipReverseLookup = false; - - @Override - public void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor) { - String[] ips = Optional.ofNullable(ipAddresses) - .filter(s -> !Strings.isNullOrEmpty(s)) - .map(s -> s.split(",")) - .orElse(new String[0]); - - NodeSpec nodeSpec = new NodeSpec(diskAvailableGb, mainMemoryAvailableGb, cpuCores, Boolean.valueOf(fastDisk), bandwidth, ips); - SpecVerificationReport specVerificationReport = verifySpec(nodeSpec, commandExecutor); - - hardwareDivergenceReport.setSpecVerificationReport(specVerificationReport); - } - - private SpecVerificationReport verifySpec(NodeSpec nodeSpec, CommandExecutor commandExecutor) { - VerifierSettings verifierSettings = new VerifierSettings(true); - HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(commandExecutor, verifierSettings); - return makeVerificationReport(actualHardware, nodeSpec, skipLookup, skipReverseLookup); - } - - private static SpecVerificationReport makeVerificationReport( - HardwareInfo actualHardware, - NodeSpec nodeSpec, - boolean skipLookup, - boolean skipReverseLookup) { - SpecVerificationReport specVerificationReport = HardwareNodeComparator.compare(NodeJsonConverter.convertJsonModelToHardwareInfo(nodeSpec), actualHardware); - IPAddressVerifier ipAddressVerifier = new IPAddressVerifier(Defaults.getDefaults().vespaHostname(), skipLookup, skipReverseLookup); - ipAddressVerifier.reportFaultyIpAddresses(nodeSpec, specVerificationReport); - return specVerificationReport; - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java deleted file mode 100644 index 3bc1a447ea8..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeSpec; - -/** - * Contains information on what spec should be verified or not. - * - * @author sgrostad - * @author olaaun - */ -public class VerifierSettings { - - private final boolean checkIPv6; - - public VerifierSettings() { - this.checkIPv6 = true; - } - - public VerifierSettings(boolean checkIPv6) { - this.checkIPv6 = checkIPv6; - } - - public VerifierSettings(NodeSpec nodeSpec) { - checkIPv6 = nodeSpec.getIpv6Address() != null; - } - - public boolean isCheckIPv6() { - return checkIPv6; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java deleted file mode 100644 index 6416babb79b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves number of CPU cores, and stores the result in a HardwareInfo instance - * - * @author olaaun - * @author sgrostad - */ -public class CPURetriever implements HardwareRetriever { - - private static final String CPU_INFO_COMMAND = "cat /proc/cpuinfo"; - private static final String SEARCH_WORD = "cpu MHz"; - private static final String REGEX_SPLIT = "\\s+:\\s"; - private static final int SEARCH_ELEMENT_INDEX = 0; - private static final int RETURN_ELEMENT_INDEX = 1; - private static final Logger logger = Logger.getLogger(CPURetriever.class.getName()); - private final HardwareInfo hardwareInfo; - private final CommandExecutor commandExecutor; - - public CPURetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { - this.hardwareInfo = hardwareInfo; - this.commandExecutor = commandExecutor; - } - - @Override - public void updateInfo() { - try { - List<String> commandOutput = commandExecutor.executeCommand(CPU_INFO_COMMAND); - List<ParseResult> parseResults = parseCPUInfoFile(commandOutput); - setCpuCores(parseResults); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve CPU info", e); - } - } - - List<ParseResult> parseCPUInfoFile(List<String> commandOutput) { - List<String> searchWords = Collections.singletonList(SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords); - return OutputParser.parseOutput(parseInstructions, commandOutput); - } - - void setCpuCores(List<ParseResult> parseResults) { - hardwareInfo.setMinCpuCores(countCpuCores(parseResults)); - } - - private int countCpuCores(List<ParseResult> parseResults) { - return parseResults.size(); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java deleted file mode 100644 index 2eb6bf80bc5..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves disk space and type, and stores the result in a HardwareInfo instance - * - * @author olaaun - * @author sgrostad - */ -public class DiskRetriever implements HardwareRetriever { - - private static final String DISK_CHECK_TYPE = "lsblk -d -o name,rota"; - private static final String DISK_CHECK_SIZE = "pvdisplay --units G | grep 'PV Size'"; - private static final String DISK_NAME = "sda"; - private static final String DISK_TYPE_REGEX_SPLIT = "\\s+"; - private static final int DISK_TYPE_SEARCH_ELEMENT_INDEX = 0; - private static final int DISK_TYPE_RETURN_ELEMENT_INDEX = 1; - private static final String DISK_SIZE_SEARCH_WORD = "Size"; - private static final String DISK_SIZE_REGEX_SPLIT = "\\s+"; - private static final int DISK_SIZE_SEARCH_ELEMENT_INDEX = 1; - private static final int DISK_SIZE_RETURN_ELEMENT_INDEX = 2; - private static final Logger logger = Logger.getLogger(DiskRetriever.class.getName()); - private final HardwareInfo hardwareInfo; - private final CommandExecutor commandExecutor; - - - public DiskRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { - this.hardwareInfo = hardwareInfo; - this.commandExecutor = commandExecutor; - } - - @Override - public void updateInfo() { - updateDiskType(); - updateDiskSize(); - } - - void updateDiskType() { - try { - List<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_TYPE); - ParseResult parseResult = parseDiskType(commandOutput); - setDiskType(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve disk type", e); - } - } - - void updateDiskSize() { - try { - List<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_SIZE); - List<ParseResult> parseResult = parseDiskSize(commandOutput); - setDiskSize(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve disk size", e); - } - } - - ParseResult parseDiskType(List<String> commandOutput) throws IOException { - List<String> searchWords = Collections.singletonList(DISK_NAME); - ParseInstructions parseInstructions = new ParseInstructions(DISK_TYPE_SEARCH_ELEMENT_INDEX, DISK_TYPE_RETURN_ELEMENT_INDEX, DISK_TYPE_REGEX_SPLIT, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput) - .orElseThrow(() -> new IOException("Parsing for disk type failed")); - } - - void setDiskType(ParseResult parseResult) { - hardwareInfo.setDiskType(DiskType.UNKNOWN); - String fastDiskSymbol = "0"; - String nonFastDiskSymbol = "1"; - if (parseResult.getValue().equals(fastDiskSymbol)) { - hardwareInfo.setDiskType(DiskType.FAST); - } else if (parseResult.getValue().equals(nonFastDiskSymbol)) { - hardwareInfo.setDiskType(DiskType.SLOW); - } - } - - List<ParseResult> parseDiskSize(List<String> commandOutput) { - List<String> searchWords = Collections.singletonList(DISK_SIZE_SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords); - return OutputParser.parseOutput(parseInstructions, commandOutput); - } - - private void setDiskSize(List<ParseResult> parseResults) { - double diskSize = 0; - try { - for (ParseResult parseResult : parseResults) { - String sizeValue = parseResult.getValue().replaceAll("[^\\d.]", ""); - diskSize += Double.parseDouble(sizeValue); - } - } catch (NumberFormatException | NullPointerException e) { - logger.log(Level.WARNING, "Parse results contained an invalid PV size - ", parseResults); - } finally { - hardwareInfo.setMinDiskAvailableGb(diskSize); - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java deleted file mode 100644 index 1c17d73523b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -/** - * All information the different retrievers retrieve is stored as a HardwareInfo object. - * - * @author olaaun - * @author sgrostad - */ -// TODO: This should be immutable -public class HardwareInfo { - - private double minDiskAvailableGb; - private double minMainMemoryAvailableGb; - private int minCpuCores; - private boolean ipv4Interface; - private boolean ipv6Interface; - private boolean ipv6Connection; - private double interfaceSpeedMbs; - private DiskType diskType; - - public double getInterfaceSpeedMbs() { - return interfaceSpeedMbs; - } - - public void setInterfaceSpeedMbs(double interfaceSpeedMbs) { - this.interfaceSpeedMbs = interfaceSpeedMbs; - } - - public double getMinDiskAvailableGb() { - return minDiskAvailableGb; - } - - public void setMinDiskAvailableGb(double minDiskAvailableGb) { - this.minDiskAvailableGb = minDiskAvailableGb; - } - - public boolean getIpv6Interface() { - return ipv6Interface; - } - - public void setIpv6Interface(boolean ipv6Interface) { - this.ipv6Interface = ipv6Interface; - } - - public boolean getIpv4Interface() { - return ipv4Interface; - } - - public void setIpv4Interface(boolean ipv4Interface) { - this.ipv4Interface = ipv4Interface; - } - - public boolean isIpv6Connection() { - return ipv6Connection; - } - - public void setIpv6Connection(boolean ipv6Connection) { - this.ipv6Connection = ipv6Connection; - } - - public double getMinMainMemoryAvailableGb() { - return minMainMemoryAvailableGb; - } - - public void setMinMainMemoryAvailableGb(double minMainMemoryAvailableGb) { - this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; - } - - public void setDiskType(DiskType diskType) { - this.diskType = diskType; - } - - public DiskType getDiskType() { - return diskType; - } - - public int getMinCpuCores() { - return minCpuCores; - } - - public void setMinCpuCores(int minCpuCores) { - this.minCpuCores = minCpuCores; - } - - public enum DiskType {SLOW, FAST, UNKNOWN} - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java deleted file mode 100644 index ba29f07baeb..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; - -import java.util.ArrayList; -import java.util.List; - -/** - * Makes a HardwareInfo object and calls all the retrievers for this object. - * - * @author olaaun - * @author sgrostad - */ -public class HardwareInfoRetriever { - - public static HardwareInfo retrieve(CommandExecutor commandExecutor, VerifierSettings verifierSettings) { - HardwareInfo hardwareInfo = new HardwareInfo(); - List<HardwareRetriever> infoList = new ArrayList<>(); - infoList.add(new CPURetriever(hardwareInfo, commandExecutor)); - infoList.add(new MemoryRetriever(hardwareInfo, commandExecutor)); - infoList.add(new DiskRetriever(hardwareInfo, commandExecutor)); - infoList.add(new NetRetriever(hardwareInfo, commandExecutor, verifierSettings)); - - for (HardwareRetriever hardwareInfoType : infoList) { - hardwareInfoType.updateInfo(); - } - return hardwareInfo; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java deleted file mode 100644 index fe02262e80f..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -/** - * @author olaaun - * @author sgrostad - */ -interface HardwareRetriever { - - /** - * Should retrieve spec from some part of the hardware, and store the result in hardwareinfo instance passed to class - */ - void updateInfo(); - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java deleted file mode 100644 index 2f5bbfc48dd..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves memory size, and stores the result in a HardwareInfo instance - * - * @author olaaun - * @author sgrostad - */ -public class MemoryRetriever implements HardwareRetriever { - - private static final String MEMORY_INFO_COMMAND = "cat /proc/meminfo"; - private static final String SEARCH_WORD = "MemTotal"; - private static final String REGEX_SPLIT = ":\\s"; - private static final int SEARCH_ELEMENT_INDEX = 0; - private static final int RETURN_ELEMENT_INDEX = 1; - private static final Logger logger = Logger.getLogger(MemoryRetriever.class.getName()); - private final HardwareInfo hardwareInfo; - private final CommandExecutor commandExecutor; - - public MemoryRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { - this.hardwareInfo = hardwareInfo; - this.commandExecutor = commandExecutor; - } - - @Override - public void updateInfo() { - try { - List<String> commandOutput = commandExecutor.executeCommand(MEMORY_INFO_COMMAND); - ParseResult parseResult = parseMemInfoFile(commandOutput); - updateMemoryInfo(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve memory info. ", e); - } - } - - ParseResult parseMemInfoFile(List<String> commandOutput) throws IOException { - List<String> searchWords = Collections.singletonList(SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput) - .orElseThrow(() -> new IOException("Failed to parse memory info file.")); - } - - void updateMemoryInfo(ParseResult parseResult) { - double memory = convertKBToGB(parseResult.getValue()); - hardwareInfo.setMinMainMemoryAvailableGb(memory); - } - - double convertKBToGB(String totMem) { - String[] split = totMem.split(" "); - double value = Double.parseDouble(split[0]); - double kiloToGiga = 1000000.0; - return value / kiloToGiga; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java deleted file mode 100644 index 9f147f79934..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; -import org.apache.commons.exec.ExecuteException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves IPv4/IPv6 interface, and checks interface speed. If node should have IPv6, tries to ping6. - * The results are stored in a HardwareInfo instance - * - * @author olaaun - * @author sgrostad - */ -public class NetRetriever implements HardwareRetriever { - - // Interface commands ignores lo-, veth- and docker interfaces - private static final String NET_FIND_INTERFACE = "/sbin/ifconfig | awk 'BEGIN {RS=\"\\n\\n\"; } { if ( $1 != \"lo\" && !match($1, \"^veth\") && !match($1, \"^docker\")) {print} }'"; - private static final String NET_CHECK_INTERFACE_SPEED = "for i in $(/sbin/ifconfig | awk 'BEGIN {RS=\"\\n\\n\"; } { if ( $1 != \"lo\" && !match($1, \"^veth\") && !match($1, \"^docker\")) {print $1} }'); do /sbin/ethtool $i; done;"; - private static final String SEARCH_WORD_INTERFACE_IP4 = "inet"; - private static final String SEARCH_WORD_INTERFACE_IPV6 = "inet6"; - private static final String SEARCH_WORD_INTERFACE_SPEED = "Speed"; - private static final String INTERFACE_NAME_REGEX_SPLIT = "\\s+"; - private static final int INTERFACE_SEARCH_ELEMENT_INDEX = 0; - private static final int INTERFACE_RETURN_ELEMENT_INDEX = 0; - private static final String INTERFACE_SPEED_REGEX_SPLIT = ":"; - private static final int INTERFACE_SPEED_SEARCH_ELEMENT_INDEX = 0; - private static final int INTERFACE_SPEED_RETURN_ELEMENT_INDEX = 1; - private static final String PING_NET_COMMAND = "ping6 -c 1 -q www.yahoo.com | grep -oP '\\d+(?=% packet loss)'"; - private static final String PING_SEARCH_WORD = "\\d+\\.?\\d*"; - private static final String PING_SPLIT_REGEX_STRING = "\\s+"; - private static final int PING_SEARCH_ELEMENT_INDEX = 0; - private static final int PING_RETURN_ELEMENT_INDEX = 0; - private static final Logger logger = Logger.getLogger(NetRetriever.class.getName()); - private final HardwareInfo hardwareInfo; - private final CommandExecutor commandExecutor; - private final VerifierSettings verifierSettings; - - - public NetRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor, VerifierSettings verifierSettings) { - this.hardwareInfo = hardwareInfo; - this.commandExecutor = commandExecutor; - this.verifierSettings = verifierSettings; - } - - @Override - public void updateInfo() { - List<ParseResult> parseResults = findInterface(); - findInterfaceSpeed(parseResults); - if (verifierSettings.isCheckIPv6()) { - testPingResponse(parseResults); - } else { - hardwareInfo.setIpv6Connection(true); - } - updateHardwareInfoWithNet(parseResults); - } - - List<ParseResult> findInterface() { - List<ParseResult> parseResults = new ArrayList<>(); - try { - List<String> commandOutput = commandExecutor.executeCommand(NET_FIND_INTERFACE); - parseResults = parseNetInterface(commandOutput); - - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve net interface. ", e); - } - return parseResults; - } - - List<ParseResult> parseNetInterface(List<String> commandOutput) { - List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_IP4, SEARCH_WORD_INTERFACE_IPV6)); - ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SEARCH_ELEMENT_INDEX, INTERFACE_RETURN_ELEMENT_INDEX, INTERFACE_NAME_REGEX_SPLIT, searchWords); - return OutputParser.parseOutput(parseInstructions, commandOutput); - } - - void findInterfaceSpeed(List<ParseResult> parseResults) { - try { - List<String> commandOutput = commandExecutor.executeCommand(NET_CHECK_INTERFACE_SPEED); - ParseResult parseResult = parseInterfaceSpeed(commandOutput); - parseResults.add(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve interface speed. ", e); - } - } - - ParseResult parseInterfaceSpeed(List<String> commandOutput) throws IOException { - List<String> searchWords = Collections.singletonList(SEARCH_WORD_INTERFACE_SPEED); - ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SPEED_SEARCH_ELEMENT_INDEX, INTERFACE_SPEED_RETURN_ELEMENT_INDEX, INTERFACE_SPEED_REGEX_SPLIT, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput) - .orElseThrow(() -> new IOException("Failed to parse interface speed output.")); - } - - private void testPingResponse(List<ParseResult> parseResults) { - try { - List<String> commandOutput = commandExecutor.executeCommand(PING_NET_COMMAND); - parseResults.add(parsePingResponse(commandOutput)); - } catch (ExecuteException e) { - logger.log(Level.WARNING, "Failed to execute ping6", e); - } catch (IOException e) { - logger.log(Level.WARNING, e.getMessage()); - } - } - - ParseResult parsePingResponse(List<String> commandOutput) throws IOException { - List<String> searchWords = Collections.singletonList(PING_SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(PING_SEARCH_ELEMENT_INDEX, PING_RETURN_ELEMENT_INDEX, PING_SPLIT_REGEX_STRING, searchWords); - return new ParseResult(PING_SEARCH_WORD, OutputParser.parseSingleOutput(parseInstructions, commandOutput) - .orElseThrow(()->new IOException("Failed to parse ping output.")).getValue()); - } - - void updateHardwareInfoWithNet(List<ParseResult> parseResults) { - hardwareInfo.setIpv6Interface(false); - hardwareInfo.setIpv4Interface(false); - for (ParseResult parseResult : parseResults) { - switch (parseResult.getSearchWord()) { - case SEARCH_WORD_INTERFACE_IP4: - hardwareInfo.setIpv4Interface(true); - break; - case SEARCH_WORD_INTERFACE_IPV6: - hardwareInfo.setIpv6Interface(true); - break; - case SEARCH_WORD_INTERFACE_SPEED: - double speed = convertInterfaceSpeed(parseResult.getValue()); - hardwareInfo.setInterfaceSpeedMbs(speed); - break; - case PING_SEARCH_WORD: - setIpv6Connectivity(parseResult); - break; - default: - throw new RuntimeException("Invalid ParseResult search word: " + parseResult.getSearchWord()); - } - } - } - - double convertInterfaceSpeed(String speed) { - return Double.parseDouble(speed.replaceAll("[^\\d.]", "")); - } - - void setIpv6Connectivity(ParseResult parseResult) { - String pingResponse = parseResult.getValue(); - String packetLoss = pingResponse.replaceAll("[^\\d.]", ""); - if (packetLoss.equals("")) return; - if (Double.parseDouble(packetLoss) > 99) return; - hardwareInfo.setIpv6Connection(true); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java deleted file mode 100644 index b32f6d076a7..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import com.yahoo.collections.Pair; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import com.yahoo.system.ProcessExecuter; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class CoreCollectorTest { - private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); - private final CoreCollector coreCollector = new CoreCollector(processExecuter); - - private final Path INSTALL_STATE_PATH = Paths.get("/path/to/install.state"); - private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234"); - private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program"); - private final List<String> GDB_BACKTRACE = Arrays.asList("[New Thread 2703]", - "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", - "#0 0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t printf(argv[3]);", - "#0 0x00000000004004d8 in main (argv=0x1) at main.c:4"); - - private final List<String> INSTALL_STATE = Arrays.asList("package: some_package-0.0.2", - "variable 'value'", - "ca_file /path/to/ca.pem"); - - private final List<String> RPM_PACKAGES = Arrays.asList("some_package-0.0.2", - "another_package-1.0.6", - "last_pkg-3.10_102"); - - @Rule - public TemporaryFolder folder= new TemporaryFolder(); - - private void mockExec(String[] cmd, String output) throws IOException { - mockExec(cmd, output, ""); - } - - private void mockExec(String[] cmd, String output, String error) throws IOException { - when(processExecuter.exec(cmd)).thenReturn(new Pair<>(error.isEmpty() ? 0 : 1, output + error)); - } - - static final String GDB_PATH = getDefaults().underVespaHome("bin64/gdb"); - - @Test - public void extractsBinaryPathTest() throws IOException { - final String[] cmd = {"file", TEST_CORE_PATH.toString()}; - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program --foo --bar baz'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin//program'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, " + - "from 'program', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, " + - "execfn: '/usr/bin/program', platform: 'x86_64"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH)); - - - Path fallbackResponse = Paths.get("/response/from/fallback"); - mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}, - "Core was generated by `/response/from/fallback'."); - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style"); - assertEquals(fallbackResponse, coreCollector.readBinPath(TEST_CORE_PATH)); - - mockExec(cmd, "", "Error code 1234"); - assertEquals(fallbackResponse, coreCollector.readBinPath(TEST_CORE_PATH)); - } - - @Test - public void extractsBinaryPathUsingGdbTest() throws IOException { - final String[] cmd = new String[]{"/bin/sh", "-c", - GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}; - - mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'."); - assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(TEST_CORE_PATH)); - - mockExec(cmd, "", "Error 123"); - try { - coreCollector.readBinPathFallback(TEST_CORE_PATH); - fail("Expected not to be able to get bin path"); - } catch (RuntimeException e) { - assertEquals("Failed to extract binary path from GDB, result: (1,Error 123), command: " + - "[/bin/sh, -c, /opt/vespa/bin64/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage()); - } - } - - @Test - public void extractsBacktraceUsingGdb() throws IOException { - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, false)); - - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - "", "Failure"); - try { - coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, false); - fail("Expected not to be able to read backtrace"); - } catch (RuntimeException e) { - assertEquals("Failed to read backtrace (1,Failure), Command: " + - "[/opt/vespa/bin64/gdb, -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage()); - } - } - - @Test - public void extractsBacktraceFromAllThreadsUsingGdb() throws IOException { - mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch", - "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, true)); - } - - @Test - public void collectsDataTest() throws IOException { - mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch", - "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{"cat", INSTALL_STATE_PATH.toString()}, String.join("\n", INSTALL_STATE)); - mockExec(new String[]{"rpm", "-qa"}, String.join("\n", RPM_PACKAGES)); - - Map<String, Object> expectedData = new HashMap<>(); - expectedData.put("bin_path", TEST_BIN_PATH.toString()); - expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE)); - expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE)); - expectedData.put("install_state", new ArrayList<>(INSTALL_STATE)); - expectedData.put("rpm_packages", new ArrayList<>(RPM_PACKAGES)); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(INSTALL_STATE_PATH))); - } - - @Test - public void collectsPartialIfUnableToDetermineDumpingProgramTest() throws IOException { - // We fail to get backtrace and RPM packages, but install state works, make sure it is returned - mockExec(new String[]{"cat", INSTALL_STATE_PATH.toString()}, String.join("\n", INSTALL_STATE)); - - Map<String, Object> expectedData = new HashMap<>(); - expectedData.put("install_state", new ArrayList<>(INSTALL_STATE)); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(INSTALL_STATE_PATH))); - } - - @Test - public void collectsPartialIfBacktraceFailsTest() throws IOException { - mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"}, - "", "Failure"); - - Map<String, Object> expectedData = new HashMap<>(); - expectedData.put("bin_path", TEST_BIN_PATH.toString()); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.empty())); - } - - @Test - public void parseTotalMemoryTestTest() { - String memInfo = "MemTotal: 100000000 kB\nMemUsed: 1000000 kB\n"; - assertEquals(100000000, coreCollector.parseTotalMemorySize(memInfo)); - - String badMemInfo = "This string has no memTotal value"; - try { - coreCollector.parseTotalMemorySize(badMemInfo); - fail("Expected to fail on parsing"); - } catch (RuntimeException e) { - assertEquals("Could not parse meminfo: " + badMemInfo, e.getMessage()); - } - } - - @Test - public void testDeleteUncompressedFiles() throws IOException { - final String documentId = "UIDD-ABCD-EFGH"; - final String coreDumpFilename = "core.dump"; - - Path coredumpPath = folder.newFolder("crash").toPath() - .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME) - .resolve(documentId); - coredumpPath.toFile().mkdirs(); - coredumpPath.resolve(coreDumpFilename).toFile().createNewFile(); - - Set<Path> expectedContentsOfCoredump = new HashSet<>(Arrays.asList( - coredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME), - coredumpPath.resolve(coreDumpFilename + ".lz4"))); - expectedContentsOfCoredump.forEach(path -> { - try { - path.toFile().createNewFile(); - } catch (IOException ignored) { ignored.printStackTrace();} - }); - coreCollector.deleteDecompressedCoredump(coredumpPath.resolve(coreDumpFilename)); - - assertEquals(expectedContentsOfCoredump, Files.list(coredumpPath).collect(Collectors.toSet())); - } - - @Test - public void testDeleteUncompressedFilesWithoutLz4() throws IOException { - final String documentId = "UIDD-ABCD-EFGH"; - final String coreDumpFilename = "core.dump"; - - Path coredumpPath = folder.newFolder("crash").toPath() - .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME) - .resolve(documentId); - coredumpPath.toFile().mkdirs(); - - Set<Path> expectedContentsOfCoredump = new HashSet<>(Arrays.asList( - coredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME), - coredumpPath.resolve(coreDumpFilename))); - expectedContentsOfCoredump.forEach(path -> { - try { - path.toFile().createNewFile(); - } catch (IOException ignored) { ignored.printStackTrace();} - }); - coreCollector.deleteDecompressedCoredump(coredumpPath.resolve(coreDumpFilename)); - - assertEquals(expectedContentsOfCoredump, Files.list(coredumpPath).collect(Collectors.toSet())); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java deleted file mode 100644 index a0125e78ca2..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import org.apache.http.HttpHeaders; -import org.apache.http.HttpVersion; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicStatusLine; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.ArgumentCaptor; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class CoredumpHandlerTest { - - private final CloseableHttpClient httpClient = mock(CloseableHttpClient.class); - private final CoreCollector coreCollector = mock(CoreCollector.class); - private static final Map<String, Object> attributes = new LinkedHashMap<>(); - private static final Map<String, Object> metadata = new LinkedHashMap<>(); - private static final String expectedMetadataFileContents = "{\"fields\":{" + - "\"bin_path\":\"/bin/bash\"," + - "\"backtrace\":[\"call 1\",\"function 2\",\"something something\"]," + - "\"hostname\":\"host123.yahoo.com\"," + - "\"vespa_version\":\"6.48.4\"," + - "\"kernel_version\":\"2.6.32-573.22.1.el6.YAHOO.20160401.10.x86_64\"," + - "\"docker_image\":\"vespa/ci:6.48.4\"}}"; - private static final String feedEndpoint = "http://feed-endpoint.hostname.tld/feed"; - - static { - attributes.put("hostname", "host123.yahoo.com"); - attributes.put("vespa_version", "6.48.4"); - attributes.put("kernel_version", "2.6.32-573.22.1.el6.YAHOO.20160401.10.x86_64"); - attributes.put("docker_image", "vespa/ci:6.48.4"); - - metadata.put("bin_path", "/bin/bash"); - metadata.put("backtrace", Arrays.asList("call 1", "function 2", "something something")); - } - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private CoredumpHandler coredumpHandler; - private Path crashPath; - private Path donePath; - - @Before - public void setup() throws IOException { - crashPath = folder.newFolder("crash").toPath(); - donePath = folder.newFolder("done").toPath(); - - coredumpHandler = new CoredumpHandler(httpClient, coreCollector, crashPath, donePath, attributes, - Optional.empty(), feedEndpoint); - } - - @Test - public void ignoresIncompleteCoredumps() throws IOException { - Path coredumpPath = createCoredump(".core.dump", Instant.now()); - Path processingPath = coredumpHandler.enqueueCoredumps(); - - // The 'processing' directory should be empty - assertFolderContents(processingPath); - - // The 'crash' directory should have 'processing' and the incomplete core dump in it - assertFolderContents(crashPath, processingPath.getFileName().toString(), coredumpPath.getFileName().toString()); - } - - @Test - public void startProcessingTest() throws IOException { - Path coredumpPath = createCoredump("core.dump", Instant.now()); - Path processingPath = crashPath.resolve("processing_dir"); - coredumpHandler.enqueueCoredumpForProcessing(coredumpPath, processingPath); - - // Contents of 'crash' should be only the 'processing' directory - assertFolderContents(crashPath, processingPath.getFileName().toString()); - - // The 'processing' directory should have 1 directory inside for the core.dump we just created - List<Path> processedCoredumps = Files.list(processingPath).collect(Collectors.toList()); - assertEquals(processedCoredumps.size(), 1); - - // Inside the coredump directory, there should be 1 file: core.dump - assertFolderContents(processedCoredumps.get(0), coredumpPath.getFileName().toString()); - } - - @Test - public void limitToProcessingOneCoredumpAtTheTimeTest() throws IOException { - final String oldestCoredump = "core.dump0"; - final Instant startTime = Instant.now(); - createCoredump(oldestCoredump, startTime.minusSeconds(3600)); - createCoredump("core.dump1", startTime.minusSeconds(1000)); - createCoredump("core.dump2", startTime); - Path processingPath = coredumpHandler.enqueueCoredumps(); - - List<Path> processingCoredumps = Files.list(processingPath).collect(Collectors.toList()); - assertEquals(1, processingCoredumps.size()); - - // Make sure that the 1 coredump that we are processing is the oldest one - Set<String> filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0)) - .map(file -> file.getFileName().toString()) - .collect(Collectors.toSet()); - assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory); - - // Running enqueueCoredumps should not start processing any new coredumps as we already are processing one - coredumpHandler.enqueueCoredumps(); - assertEquals(processingCoredumps, Files.list(processingPath).collect(Collectors.toList())); - filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0)) - .map(file -> file.getFileName().toString()) - .collect(Collectors.toSet()); - assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory); - } - - @Test - public void coredumpMetadataCollectAndWriteTest() throws IOException { - createCoredump("core.dump", Instant.now()); - Path processingPath = coredumpHandler.enqueueCoredumps(); - Path processingCoredumpPath = Files.list(processingPath).findFirst().orElseThrow(() -> - new RuntimeException("Expected to find directory with coredump in processing dir")); - when(coreCollector.collect(eq(processingCoredumpPath.resolve("core.dump")), any())).thenReturn(metadata); - - // Inside 'processing' directory, there should be a new directory containing 'core.dump' file - String returnedMetadata = coredumpHandler.collectMetadata(processingCoredumpPath, attributes); - String metadataFileContents = new String(Files.readAllBytes( - processingCoredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME))); - assertEquals(expectedMetadataFileContents, metadataFileContents); - assertEquals(expectedMetadataFileContents, returnedMetadata); - } - - @Test - public void coredumpMetadataReadIfExistsTest() throws IOException { - final String documentId = "UIDD-ABCD-EFGH"; - Path metadataPath = createProcessedCoredump(documentId); - - verifyZeroInteractions(coreCollector); - String returnedMetadata = coredumpHandler.collectMetadata(metadataPath.getParent(), attributes); - assertEquals(expectedMetadataFileContents, returnedMetadata); - } - - @Test - public void reportSuccessCoredumpTest() throws IOException, URISyntaxException { - final String documentId = "UIDD-ABCD-EFGH"; - Path coredumpPath = createProcessedCoredump(documentId); - - setNextHttpResponse(200, Optional.empty()); - coredumpHandler.report(coredumpPath.getParent(), expectedMetadataFileContents); - validateNextHttpPost(documentId, expectedMetadataFileContents); - } - - @Test - public void reportFailCoredumpTest() throws IOException, URISyntaxException { - final String documentId = "UIDD-ABCD-EFGH"; - Path metadataPath = createProcessedCoredump(documentId); - - setNextHttpResponse(500, Optional.of("Internal server error")); - coredumpHandler.processAndReportCoredumps(crashPath.resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)); - validateNextHttpPost(documentId, expectedMetadataFileContents); - - // The coredump should not have been moved out of 'processing' and into 'done' as the report failed - assertFolderContents(donePath); - assertFolderContents(metadataPath.getParent(), CoredumpHandler.METADATA_FILE_NAME); - } - - @Test - public void finishProcessingTest() throws IOException { - final String documentId = "UIDD-ABCD-EFGH"; - - Path coredumpPath = createProcessedCoredump(documentId); - coredumpHandler.finishProcessing(coredumpPath.getParent()); - - // The coredump should've been moved out of 'processing' and into 'done' - assertFolderContents(crashPath.resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)); - assertFolderContents(donePath.resolve(documentId), CoredumpHandler.METADATA_FILE_NAME); - } - - - private static void assertFolderContents(Path pathToFolder, String... filenames) throws IOException { - Set<Path> expectedContentsOfFolder = Arrays.stream(filenames) - .map(pathToFolder::resolve) - .collect(Collectors.toSet()); - Set<Path> actualContentsOfFolder = Files.list(pathToFolder).collect(Collectors.toSet()); - assertEquals(expectedContentsOfFolder, actualContentsOfFolder); - } - - private Path createCoredump(String coredumpName, Instant lastModified) throws IOException { - Path coredumpPath = crashPath.resolve(coredumpName); - coredumpPath.toFile().createNewFile(); - coredumpPath.toFile().setLastModified(lastModified.toEpochMilli()); - return coredumpPath; - } - - private Path createProcessedCoredump(String documentId) throws IOException { - Path coredumpPath = crashPath - .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME) - .resolve(documentId) - .resolve(CoredumpHandler.METADATA_FILE_NAME); - coredumpPath.getParent().toFile().mkdirs(); - return Files.write(coredumpPath, expectedMetadataFileContents.getBytes()); - } - - private void setNextHttpResponse(int code, Optional<String> message) throws IOException { - CloseableHttpResponse response = mock(CloseableHttpResponse.class); - when(response.getStatusLine()).thenReturn(new BasicStatusLine(HttpVersion.HTTP_1_1, code, null)); - if (message.isPresent()) when(response.getEntity()).thenReturn(new StringEntity(message.get())); - - when(httpClient.execute(any())).thenReturn(response); - } - - private void validateNextHttpPost(String documentId, String expectedBody) throws IOException, URISyntaxException { - ArgumentCaptor<HttpPost> capturedPost = ArgumentCaptor.forClass(HttpPost.class); - verify(httpClient).execute(capturedPost.capture()); - - URI expectedURI = new URI(feedEndpoint + "/" + documentId); - assertEquals(expectedURI, capturedPost.getValue().getURI()); - assertEquals("application/json", capturedPost.getValue().getHeaders(HttpHeaders.CONTENT_TYPE)[0].getValue()); - assertEquals(expectedBody, - new BufferedReader(new InputStreamReader(capturedPost.getValue().getEntity().getContent())).readLine()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java deleted file mode 100644 index eefdcf4adad..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author freva - */ -public class FileHelperTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void initFiles() throws IOException { - for (int i=0; i<10; i++) { - File temp = folder.newFile("test_" + i + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis()); - } - - for (int i=0; i<7; i++) { - File temp = folder.newFile("test_" + i + "_file.test"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis()); - } - - for (int i=0; i<5; i++) { - File temp = folder.newFile(i + "-abc" + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis()); - } - - File temp = folder.newFile("week_old_file.json"); - temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis()); - } - - @Test - public void testDeleteAll() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.empty(), false); - - assertEquals(0, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(6, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file - } - - @Test - public void testDeleteSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".json$"), false); - - assertEquals(7, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefixAndSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 7 test_*_file.test files + week_old_file - } - - @Test - public void testDeleteOld() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(600), Optional.empty(), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file - } - - @Test - public void testDeleteWithAllParameters() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(200), Optional.of("^test_.*\\.json$"), false); - - assertEquals(15, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 8 (from test_*_.json) - } - - @Test - public void testDeleteWithSubDirectoriesNoRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - // 6 test_*.json from test_folder1/ - // + 9 test_*.json and 4 abc_*.json from test_folder2/ - // + 13 test_*.json from test_folder2/subSubFolder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves - assertEquals(48, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteWithSubDirectoriesRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), true); - - // 4 abc_*.json from test_folder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder2/ itself - assertEquals(18, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(8, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file + 2 directories - } - - @Test - public void testGetContentsOfNonExistingDirectory() { - Path fakePath = Paths.get("/some/made/up/dir/"); - assertEquals(Collections.emptyList(), FileHelper.listContentsOfDirectory(fakePath)); - } - - @Test(expected=IllegalArgumentException.class) - public void testDeleteFilesExceptNMostRecentWithNegativeN() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), -5); - } - - @Test - public void testDeleteFilesExceptFiveMostRecent() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 5); - - assertEquals(5, getContentsOfDirectory(folder.getRoot()).length); - - String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"}; - String[] remainingFiles = Arrays.stream(getContentsOfDirectory(folder.getRoot())) - .map(File::getName) - .sorted() - .toArray(String[]::new); - - assertArrayEquals(oldestFiles, remainingFiles); - } - - @Test - public void testDeleteFilesExceptNMostRecentWithLargeN() throws IOException { - String[] filesPreDelete = folder.getRoot().list(); - - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 50); - - assertArrayEquals(filesPreDelete, folder.getRoot().list()); - } - - @Test - public void testDeleteFilesLargerThan10B() throws IOException { - initSubDirectories(); - - File temp1 = new File(folder.getRoot(), "small_file"); - writeNBytesToFile(temp1, 50); - - File temp2 = new File(folder.getRoot(), "some_file"); - writeNBytesToFile(temp2, 20); - - File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file"); - writeNBytesToFile(temp3, 75); - - FileHelper.deleteFilesLargerThan(folder.getRoot().toPath(), 10); - - assertEquals(58, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - assertFalse(temp1.exists() || temp2.exists() || temp3.exists()); - } - - @Test - public void testDeleteDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".*folder2")); - - //23 files in root - // + 6 in test_folder1 + test_folder1 itself - assertEquals(30, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteDirectoriesBasedOnAge() throws IOException { - initSubDirectories(); - // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is - // also older than maxAge inside the sub directory, create some files which are newer than maxAge. - // deleteDirectories() should NOT delete folder3 - File subFolder3 = folder.newFolder("test_folder3"); - File subSubFolder3 = folder.newFolder("test_folder3", "subSubFolder3"); - - for (int j=0; j<11; j++) { - File.createTempFile("test_", ".json", subSubFolder3); - } - - subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*")); - - //23 files in root - // + 13 in test_folder2 - // + 13 in subSubFolder2 - // + 11 in subSubFolder3 - // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself - assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testRecursivelyDeleteDirectory() throws IOException { - initSubDirectories(); - FileHelper.recursiveDelete(folder.getRoot().toPath()); - assertFalse(folder.getRoot().exists()); - } - - @Test - public void testRecursivelyDeleteRegularFile() throws IOException { - File file = folder.newFile(); - assertTrue(file.exists()); - assertTrue(file.isFile()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testRecursivelyDeleteNonExistingFile() throws IOException { - File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile(); - assertFalse(file.exists()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testInitSubDirectories() throws IOException { - initSubDirectories(); - assertTrue(folder.getRoot().exists()); - assertTrue(folder.getRoot().isDirectory()); - - Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1"); - assertTrue(test_folder1.toFile().exists()); - assertTrue(test_folder1.toFile().isDirectory()); - - Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2"); - assertTrue(test_folder2.toFile().exists()); - assertTrue(test_folder2.toFile().isDirectory()); - - Path subSubFolder2 = test_folder2.resolve("subSubFolder2"); - assertTrue(subSubFolder2.toFile().exists()); - assertTrue(subSubFolder2.toFile().isDirectory()); - } - - @Test - public void testDoesNotFailOnLastModifiedOnSymLink() throws IOException { - Path symPath = folder.getRoot().toPath().resolve("symlink"); - Path fakePath = Paths.get("/some/not/existant/file"); - - Files.createSymbolicLink(symPath, fakePath); - assertTrue(Files.isSymbolicLink(symPath)); - assertFalse(Files.exists(fakePath)); - - // Not possible to set modified time on symlink in java, so just check that it doesn't crash - FileHelper.getLastModifiedTime(symPath).toInstant(); - } - - private void initSubDirectories() throws IOException { - File subFolder1 = folder.newFolder("test_folder1"); - File subFolder2 = folder.newFolder("test_folder2"); - File subSubFolder2 = folder.newFolder("test_folder2", "subSubFolder2"); - - for (int j=0; j<6; j++) { - File temp = File.createTempFile("test_", ".json", subFolder1); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis()); - } - - for (int j=0; j<9; j++) { - File.createTempFile("test_", ".json", subFolder2); - } - - for (int j=0; j<4; j++) { - File.createTempFile("abc_", ".txt", subFolder2); - } - - for (int j=0; j<13; j++) { - File temp = File.createTempFile("test_", ".json", subSubFolder2); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis()); - } - - //Must be after all the files have been created - subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis()); - subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - } - - private static int getNumberOfFilesAndDirectoriesIn(File folder) { - int total = 0; - for (File file : folder.listFiles()) { - if (file.isDirectory()) { - total += getNumberOfFilesAndDirectoriesIn(file); - } - total++; - } - - return total; - } - - private static void writeNBytesToFile(File file, int nBytes) throws IOException { - Files.write(file.toPath(), new byte[nBytes]); - } - - - static File[] getContentsOfDirectory(File directory) { - File[] directoryContents = directory.listFiles(); - - return directoryContents == null ? new File[0] : directoryContents; - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java deleted file mode 100644 index 5f5aca825c0..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons; - -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class CommandExecutorTest { - - private CommandExecutor commandExecutor; - - @Before - public void setup() { - commandExecutor = new CommandExecutor(); - } - - @Test - public void test_if_executeAString_reads_testReadFile_correct() throws IOException { - String command = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt"; - List<String> commandOutput = commandExecutor.executeCommand(command); - List<String> expectedOutput = asList("This test file tests apache commons exec", "Second line"); - assertEquals(expectedOutput, commandOutput); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java deleted file mode 100644 index c30afbf4fe5..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.noderepo; - -import org.junit.Before; -import org.junit.Test; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Arrays; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.spy; - -/** - * @author sgrostad - * @author olaaun - */ -public class IPAddressVerifierTest { - - private final String ipv4Address = "10.2.4.8"; - private final String ipv6Address = "fdab:0:0:0:0:0:0:1234"; - private final NodeSpec nodeSpec = new NodeSpec(1920, 256, 48, true, 10_000, new String[]{ipv4Address, ipv6Address}); - private final String hostname = "test123.region.domain.tld"; - - private IPAddressVerifier ipAddressVerifier = spy(new IPAddressVerifier(hostname, false, false)); - private String ipv4LookupFormat; - private String ipv6LookupFormat; - - @Before - public void setup() { - ipv4LookupFormat = "8.4.2.10.in-addr.arpa"; - ipv6LookupFormat = "4.3.2.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.b.a.d.f.ip6.arpa"; - } - - @Test - public void getFaultyIpAddresses_with_hostname_resolving_to_other_ips() throws Exception { - doReturn(Arrays.asList(InetAddress.getByName("1.2.3.4"), InetAddress.getByName("fd00::1"))).when(ipAddressVerifier).mockableGetAllByName(hostname); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv4Address, ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_with_hostname_not_resolving_to_ipv4_address() throws Exception { - doReturn(Arrays.asList(InetAddress.getByName(ipv6Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv4Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_with_failing_hostname_resolution() throws Exception { - doThrow(new UnknownHostException("bad hostname")).when(ipAddressVerifier).mockableGetAllByName(hostname); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv4Address, ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_with_hostname_not_resolving_to_ipv6_address() throws Exception { - doReturn(Arrays.asList(InetAddress.getByName(ipv4Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_IP_address_when_different_hostname() throws Exception { - String wrongHostName = "www.yahoo.com"; - doReturn(Arrays.asList(InetAddress.getByName(ipv4Address), InetAddress.getByName(ipv6Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - doReturn(wrongHostName).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_all_addresses_point_to_correct_hostname() throws Exception { - doReturn(Arrays.asList(InetAddress.getByName(ipv4Address), InetAddress.getByName(ipv6Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_lookup_is_skipped() throws Exception { - ipAddressVerifier = spy(new IPAddressVerifier(hostname, true, false)); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_reverse_lookup_is_skipped() throws Exception { - ipAddressVerifier = spy(new IPAddressVerifier(hostname, false, true)); - doReturn(Arrays.asList(InetAddress.getByName(ipv4Address), InetAddress.getByName(ipv6Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void convertIpv6ToLookupFormat_should_return_properly_converted_ipv6_address() { - String actualConvertedAddress = ipAddressVerifier.convertIpv6ToLookupFormat(ipv6Address); - assertEquals(ipv6LookupFormat, actualConvertedAddress); - } - - @Test - public void convertIpv4ToLookupFormat_should_return_properly_converted_ipv6_address() { - String actualConvertedAddress = ipAddressVerifier.convertIpv4ToLookupFormat(ipv4Address); - assertEquals(ipv4LookupFormat, actualConvertedAddress); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_parameters_are_invalid() { - final NodeSpec nodeWithNoIP = new NodeSpec(1920, 256, 48, true, 10_000, new String[0]); - assertEquals(0, ipAddressVerifier.getFaultyIpAddresses(nodeWithNoIP).length); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java deleted file mode 100644 index 06d34a452eb..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.parser; - -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class OutputParserTest { - - private static final String RETURN_VALUE = "#returnValue#"; - private static final String OUTPUT_WITH_MATCH_1 = "This; Should be; a match; when; Parsing ; " + RETURN_VALUE; - private static final String OUTPUT_WITHOUT_MATCH = "This; is; not a; match"; - private static final String OUTPUT_WITH_MATCH_2 = "But; thiS will-also; be; a match:; this ; " + RETURN_VALUE; - private static final String SEARCH_WORD_1 = "Parsing"; - private static final String SEARCH_WORD_2 = "this"; - private static final String REGEX_SEARCH_WORD = ".*S.*"; - private List<String> commandOutput; - private List<String> searchWords; - - @Before - public void setup() { - commandOutput = new ArrayList<>(Arrays.asList(OUTPUT_WITH_MATCH_1, OUTPUT_WITHOUT_MATCH, OUTPUT_WITH_MATCH_2)); - } - - @Test - public void parseOutput_searching_for_two_normal_words() { - searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2)); - ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); - List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); - ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); - ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE); - assertEquals(expectedParseResult1, parseResults.get(0)); - assertEquals(expectedParseResult2, parseResults.get(1)); - } - - @Test - public void parseOutput_searching_for_two_normal_words_with_semicolon_as_line_split() { - searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2)); - ParseInstructions parseInstructions = new ParseInstructions(4, 5, ";", searchWords); - List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); - ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); - ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE); - assertEquals(expectedParseResult1, parseResults.get(0)); - assertEquals(expectedParseResult2, parseResults.get(1)); - } - - @Test - public void parseOutput_searching_for_word_containing_capital_s() { - searchWords = Collections.singletonList(REGEX_SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(1, 8, " ", searchWords); - List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); - ParseResult expectedParseResult1 = new ParseResult("Should", RETURN_VALUE); - ParseResult expectedParseResult2 = new ParseResult("thiS", RETURN_VALUE); - assertEquals(expectedParseResult1, parseResults.get(0)); - assertEquals(expectedParseResult2, parseResults.get(1)); - } - - @Test - public void parseSingleOutput_should_return_first_match() { - searchWords = Collections.singletonList(SEARCH_WORD_1); - ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); - Optional<ParseResult> parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); - ParseResult expectedParseResult = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); - assertEquals(Optional.of(expectedParseResult), parseResult); - } - - @Test - public void parseSingleOutput_should_return_invalid_parseResult() { - searchWords = Collections.singletonList("No match"); - ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); - Optional<ParseResult> parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); - assertEquals(Optional.empty(), parseResult); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java deleted file mode 100644 index 0caf63594be..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -/** - * @author sgrostad - * @author olaaun - */ - -public class BenchmarkResultInspectorTest { - - private BenchmarkResults benchmarkResults; - private static final double VALID_CPU_FREQUENCY = 2.031; - private static final double INVALID_CPU_FREQUENCY = 0.1; - private static final double VALID_DISK_SPEED = 1100.0; - private static final double INVALID_DISK_SPEED = 0.1; - private static final double VALID_MEMORY_WRITE_SPEED = 1.7; - private static final double INVALID_MEMORY_WRITE_SPEED = 0.1; - private static final double VALID_MEMORY_READ_SPEED = 4.3; - private static final double INVALID_MEMORY_READ_SPEED = 0.1; - - @Before - public void setup() { - benchmarkResults = new BenchmarkResults(); - benchmarkResults.setCpuCyclesPerSec(VALID_CPU_FREQUENCY); - benchmarkResults.setDiskSpeedMbs(VALID_DISK_SPEED); - benchmarkResults.setMemoryWriteSpeedGBs(VALID_MEMORY_WRITE_SPEED); - benchmarkResults.setMemoryReadSpeedGBs(VALID_MEMORY_READ_SPEED); - } - - @Test - public void isBenchmarkResultsValid_should_return_BenchmarkReport_with_all_values_null() { - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNull(benchmarkReport.getCpuCyclesPerSec()); - assertNull(benchmarkReport.getDiskSpeedMbs()); - assertNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - - @Test - public void isBenchmarkResultsValid_should_only_set_cpu_frequency() { - benchmarkResults.setCpuCyclesPerSec(INVALID_CPU_FREQUENCY); - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNotNull(benchmarkReport.getCpuCyclesPerSec()); - assertNull(benchmarkReport.getDiskSpeedMbs()); - assertNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - - @Test - public void isBenchmarkResultsValid_should_only_set_disk_speed() { - benchmarkResults.setDiskSpeedMbs(INVALID_DISK_SPEED); - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNull(benchmarkReport.getCpuCyclesPerSec()); - assertNotNull(benchmarkReport.getDiskSpeedMbs()); - assertNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - -// @Test TODO: Temporarily disabled due to Meltdown/Spectre performance impact, see VESPA-11051 - public void isBenchmarkResultsValid_should_only_set_memory_read_speed() { - benchmarkResults.setMemoryReadSpeedGBs(INVALID_MEMORY_READ_SPEED); - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNull(benchmarkReport.getCpuCyclesPerSec()); - assertNull(benchmarkReport.getDiskSpeedMbs()); - assertNotNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - -// @Test TODO: Temporarily disabled due to Meltdown/Spectre performance impact, see VESPA-11051 - public void isBenchmarkResultsValid_should_only_set_memory_write_speed() { - benchmarkResults.setMemoryWriteSpeedGBs(INVALID_MEMORY_WRITE_SPEED); - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNull(benchmarkReport.getCpuCyclesPerSec()); - assertNull(benchmarkReport.getDiskSpeedMbs()); - assertNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNotNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java deleted file mode 100644 index 13e13f1f83f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.yahoo.vespa.hosted.node.verification.Main; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class HardwareBenchmarkerTest { - - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/"; - private static final String VALID_DISK_BENCHMARK_PATH = RESOURCE_PATH + "diskBenchmarkValidOutput"; - private static final String VALID_SLOW_DISK_BENCHMARK_PATH = RESOURCE_PATH + "diskBenchmarkValidSlowOutput"; - private static final String VALID_CPU_BENCHMARK_PATH = RESOURCE_PATH + "cpuCyclesWithCommasTimeWithDotTest.txt"; - private static final String VALID_MEMORY_WRITE_BENCHMARK_PATH = RESOURCE_PATH + "validMemoryWriteSpeed"; - private static final String VALID_MEMORY_READ_BENCHMARK_PATH = RESOURCE_PATH + "validMemoryReadSpeed"; - private final MockCommandExecutor commandExecutor = new MockCommandExecutor(); - - @Test - public void benchmark_with_no_failures() throws Exception { - commandExecutor.addCommand("cat " + VALID_DISK_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_CPU_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - commandExecutor.addCommand("cat " + VALID_MEMORY_WRITE_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_MEMORY_READ_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - - String result = Main.execute(new String[] { - "benchmark", - }, commandExecutor); - - assertEquals("null", result); - } - - @Test - public void disk_benchmark_failure() throws Exception { - commandExecutor.addCommand("cat " + VALID_SLOW_DISK_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_CPU_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - commandExecutor.addCommand("cat " + VALID_MEMORY_WRITE_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_MEMORY_READ_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - - String result = Main.execute(new String[]{ - "benchmark", - }, commandExecutor); - - assertEquals("{\"benchmarkReport\":{\"diskSpeedMbs\":49.0}}", result); - } - - - @Test - public void preserve_previous_spec_verifier_result() throws Exception { - commandExecutor.addCommand("cat " + VALID_DISK_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_CPU_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - commandExecutor.addCommand("cat " + VALID_MEMORY_WRITE_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_MEMORY_READ_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - - final String previousResult = "{\"specVerificationReport\":{\"actualMemoryAvailable\":4.042128}}"; - - String result = Main.execute(new String[] { - "benchmark", - "-h", previousResult - }, commandExecutor); - - assertEquals(previousResult, result); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java deleted file mode 100644 index 70fab6eecb1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author sgrostad - * @author olaaun - */ - -public class CPUBenchmarkTest { - - private static final String cpuEuropeanDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt"; - private static final String cpuAlternativeDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt"; - private static final String cpuWrongOutput = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt"; - private BenchmarkResults benchmarkResults; - private MockCommandExecutor commandExecutor; - private CPUBenchmark cpu; - private static final double DELTA = 0.1; - - @Before - public void setup() { - commandExecutor = new MockCommandExecutor(); - benchmarkResults = new BenchmarkResults(); - cpu = new CPUBenchmark(benchmarkResults, commandExecutor); - } - - @Test - public void doBenchmark_find_correct_cpuCyclesPerSec() { - String mockCommand = "cat " + cpuAlternativeDelimiters; - commandExecutor.addCommand(mockCommand); - cpu.doBenchmark(); - double result = benchmarkResults.getCpuCyclesPerSec(); - double expected = 2.1576482291815062; - assertEquals(expected, result, DELTA); - } - - @Test - public void doBenchmark_wrong_output_stores_frequency_of_zero() { - String mockCommand = "cat " + cpuWrongOutput; - commandExecutor.addCommand(mockCommand); - cpu.doBenchmark(); - double result = benchmarkResults.getCpuCyclesPerSec(); - double expected = 0; - assertEquals(expected, result, DELTA); - } - - @Test - public void parseCpuCyclesPerSec_return_correct_ArrayList() throws IOException { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(cpuEuropeanDelimiters); - List<ParseResult> parseResults = cpu.parseCpuCyclesPerSec(mockCommandOutput); - ParseResult expectedParseCyclesResult = new ParseResult("cycles", "2.066.201.729"); - ParseResult expectedParseSecondsResult = new ParseResult("seconds", "0,957617512"); - assertEquals(expectedParseCyclesResult, parseResults.get(0)); - assertEquals(expectedParseSecondsResult, parseResults.get(1)); - } - - @Test - public void test_if_setCpuCyclesPerSec_reads_output_correctly() { - List<ParseResult> parseResults = new ArrayList<>(); - parseResults.add(new ParseResult("cycles", "2.066.201.729")); - parseResults.add(new ParseResult("seconds", "0,957617512")); - cpu.setCpuCyclesPerSec(parseResults); - double expectedCpuCyclesPerSec = 2.1576482291815062; - assertEquals(expectedCpuCyclesPerSec, benchmarkResults.getCpuCyclesPerSec(), DELTA); - } - - @Test - public void test_if_makeCyclesDouble_converts_European_and_alternative_delimiters_correctly() { - String toBeConvertedEuropean = "2.066.201.729"; - String toBEConvertedAlternative = "2,066,201,729"; - double expectedCycles = 2066201729; - assertEquals(expectedCycles, cpu.makeCyclesDouble(toBeConvertedEuropean), DELTA); - assertEquals(expectedCycles, cpu.makeCyclesDouble(toBEConvertedAlternative), DELTA); - } - - @Test - public void test_if_makeSecondsDouble_converts_European_and_alternative_delimiters_correctly() { - String toBeConvertedEuropean = "0,957617512"; - String toBEConvertedAlternative = "0.957617512"; - double expectedSeconds = 0.957617512; - assertEquals(expectedSeconds, cpu.makeSecondsDouble(toBeConvertedEuropean), DELTA); - assertEquals(expectedSeconds, cpu.makeSecondsDouble(toBEConvertedAlternative), DELTA); - } - - @Test - public void test_if_checkIfNumber_returns_true() { - String number = "125.5"; - assertTrue(cpu.checkIfNumber(number)); - } - - @Test - public void test_if_checkIfNumber_returns_false() { - String notANumber = "125.5a"; - assertFalse(cpu.checkIfNumber(notANumber)); - } - - @Test - public void test_if_convertToGHz_converts_correctly() { - double cycles = 2066201729; - double seconds = 0.957617512; - double expectedGHz = 2.1576482291815062; - assertEquals(expectedGHz, cpu.convertToGHz(cycles, seconds), DELTA); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java deleted file mode 100644 index f2dbfbdbb6f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author sgrostad - * @author olaaun - */ - -public class DiskBenchmarkTest { - - private DiskBenchmark diskBenchmark; - private BenchmarkResults benchmarkResults; - private MockCommandExecutor commandExecutor; - private static final String VALID_OUTPUT_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput"; - private static final String INVALID_OUTPUT_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput"; - private static final double DELTA = 0.1; - - @Before - public void setup() { - commandExecutor = new MockCommandExecutor(); - benchmarkResults = new BenchmarkResults(); - diskBenchmark = new DiskBenchmark(benchmarkResults, commandExecutor); - } - - @Test - public void doBenchmark_should_store_diskSpeed_when_valid_output() { - String mockCommand = "cat " + VALID_OUTPUT_FILE; - commandExecutor.addCommand(mockCommand); - diskBenchmark.doBenchmark(); - double expectedSpeed = 243; - double actualSpeed = benchmarkResults.getDiskSpeedMbs(); - assertEquals(expectedSpeed, actualSpeed, DELTA); - } - - @Test - public void doBenchmark_should_store_diskSpeed_as_zero_when_invalid_output() { - String mockCommand = "cat " + INVALID_OUTPUT_FILE; - commandExecutor.addCommand(mockCommand); - diskBenchmark.doBenchmark(); - double expectedSpeed = 0; - double actualSpeed = benchmarkResults.getDiskSpeedMbs(); - assertEquals(expectedSpeed, actualSpeed, DELTA); - } - - - @Test - public void parseDiskSpeed_valid_input() throws Exception { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_OUTPUT_FILE); - Optional<ParseResult> parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput); - ParseResult expectedParseResult = new ParseResult("MB/s", "243"); - assertEquals(Optional.of(expectedParseResult), parseResult); - } - - @Test - public void parseDiskSpeed_invalid_input() throws Exception { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_OUTPUT_FILE); - Optional<ParseResult> parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput); - assertEquals(Optional.empty(), parseResult); - } - - @Test - public void setDiskSpeed_valid_input() { - ParseResult parseResult = new ParseResult("MB/s", "243"); - diskBenchmark.setDiskSpeed(Optional.of(parseResult)); - double expectedDiskSpeed = 243; - assertEquals(expectedDiskSpeed, benchmarkResults.getDiskSpeedMbs(), DELTA); - } - - @Test - public void setDiskSpeed_invalid_input() { - diskBenchmark.setDiskSpeed(Optional.empty()); - double expectedDiskSpeed = 0; - assertEquals(expectedDiskSpeed, benchmarkResults.getDiskSpeedMbs(), DELTA); - } - - @Test - public void getDiskSpeedInMBs_for_KBs_MBs_and_GBs() { - ParseResult KBsParseResult = new ParseResult("kB/s", "243000"); - ParseResult MBsParseResult = new ParseResult("MB/s", "243"); - ParseResult GBsParseResult = new ParseResult("GB/s", "0.243"); - double expectedMBs = 243; - assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(KBsParseResult), DELTA); - assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(MBsParseResult), DELTA); - assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(GBsParseResult), DELTA); - } - - @Test - public void ckeckSpeedValidity_should_return_true_for_valid_format() { - String speed = "123"; - assertTrue(diskBenchmark.checkSpeedValidity(speed)); - speed = "30000"; - assertTrue(diskBenchmark.checkSpeedValidity(speed)); - speed = "6"; - assertTrue(diskBenchmark.checkSpeedValidity(speed)); - } - - @Test - public void ckeckSpeedValidity_should_return_false_for_valid_format() { - String speed = "124 GHz"; - assertFalse(diskBenchmark.checkSpeedValidity(speed)); - speed = null; - assertFalse(diskBenchmark.checkSpeedValidity(speed)); - speed = "This should return false as well"; - assertFalse(diskBenchmark.checkSpeedValidity(speed)); - } - - @Test - public void convertToMbs_should_return_properly_converted_disk_speeds() { - String speed = "1234"; - double factor = 1000; - double expectedSpeed = 1234000; - assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); - factor = 1 / 1000.0; - expectedSpeed = 1.234; - assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); - factor = 1; - expectedSpeed = 1234; - assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java deleted file mode 100644 index e2dc19c4f4c..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ -public class MemoryBenchmarkTest { - - private MemoryBenchmark memoryBenchmark; - private BenchmarkResults benchmarkResults; - private MockCommandExecutor commandExecutor; - private static final double DELTA = 0.1; - - @Before - public void setup() { - benchmarkResults = new BenchmarkResults(); - commandExecutor = new MockCommandExecutor(); - memoryBenchmark = new MemoryBenchmark(benchmarkResults, commandExecutor); - } - - @Test - public void doBenchMark_should_update_read_and_write_memory_speed() { - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - commandExecutor.addCommand("cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed"); - commandExecutor.addCommand("cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed"); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - memoryBenchmark.doBenchmark(); - double expectedReadSpeed = 5.9; - double expectedWriteSpeed = 3.4; - assertEquals(expectedReadSpeed, benchmarkResults.getMemoryReadSpeedGBs(), DELTA); - assertEquals(expectedWriteSpeed, benchmarkResults.getMemoryWriteSpeedGBs(), DELTA); - } - - @Test - public void parseMemorySpeed_valid_output() { - Double expectedSpeed = 12.1; - String mockOutput = "This is a test \n the memory speed to be found is " + expectedSpeed + " GB/s"; - List<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput); - Optional<Double> parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); - assertEquals(Optional.of(12.1), parseResult); - } - - @Test - public void parseMemorySpeed_slow_but_valid_output() { - Double expectedSpeed = 960D; - String mockOutput = "This is a test \n the memory speed to be found is " + expectedSpeed + " MB/s"; - List<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput); - Optional<Double> parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); - assertEquals(Optional.of(0.96), parseResult); - } - - @Test - public void parseMemorySpeed_invalid_output() { - List<String> mockCommandOutput = commandExecutor.outputFromString(""); - Optional<Double> parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); - assertEquals(Optional.empty(), parseResult); - mockCommandOutput = commandExecutor.outputFromString("Exit status 1"); - parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); - assertEquals(Optional.empty(), parseResult); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt deleted file mode 100644 index 08782ef065f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt +++ /dev/null @@ -1,2 +0,0 @@ - 2,066,201,729 cycles - 0.957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt deleted file mode 100644 index fd58bfaf8c7..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt +++ /dev/null @@ -1,2 +0,0 @@ - 2.066.201.729 cycles - 0,957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt deleted file mode 100644 index 61a4c8fb2a2..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt +++ /dev/null @@ -1,2 +0,0 @@ - Dummy 2.066.201.729 cycles - 0,957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput deleted file mode 100644 index 4b3d2c6edfb..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput +++ /dev/null @@ -1,5 +0,0 @@ -dd: bs: illegal numeric value - -real 0m0.074s -user 0m0.002s -sys 0m0.058s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput deleted file mode 100644 index 26778d9dd9a..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput +++ /dev/null @@ -1,5 +0,0 @@ -243 MB/s - -real 0m1.448s -user 0m0.000s -sys 0m0.260s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidSlowOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidSlowOutput deleted file mode 100644 index 773850d8640..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidSlowOutput +++ /dev/null @@ -1,5 +0,0 @@ -49 MB/s - -real 0m1.448s -user 0m0.000s -sys 0m0.260s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt deleted file mode 100644 index 28858c30a70..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt +++ /dev/null @@ -1,2 +0,0 @@ -This test file tests apache commons exec -Second line
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed deleted file mode 100644 index 26657cea9e0..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed +++ /dev/null @@ -1,3 +0,0 @@ -512+0 records in -512+0 records out -536870912 bytes (537 MB) copied, 0.0904486 s, 5.9 GB/s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed deleted file mode 100644 index ab4ccc986f8..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed +++ /dev/null @@ -1,3 +0,0 @@ -512+0 records in -512+0 records out -536870912 bytes (537 MB) copied, 0.158612 s, 3.4 GB/s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java deleted file mode 100644 index 88df18e9d4b..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.mock; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * @author sgrostad - * @author olaaun - */ - -public class MockCommandExecutor extends CommandExecutor { - - private ArrayList<String> mockCommands; - private int counter; - private static final String DUMMY_COMMAND = "DUMMY"; - - public MockCommandExecutor() { - mockCommands = new ArrayList<>(); - counter = 0; - } - - @Override - public List<String> executeCommand(String command) throws IOException { - String mockCommand = mockCommands.get(counter++); - if (mockCommand.equals(DUMMY_COMMAND)) return null; - return super.executeCommand(mockCommand); - } - - public void addCommand(String command) { - mockCommands.add(command); - } - - public void addDummyCommand() { - mockCommands.add(DUMMY_COMMAND); - } - - public static List<String> readFromFile(String filepath) throws IOException { - return new ArrayList<>(Arrays.asList(new String(Files.readAllBytes(Paths.get(filepath))).split("\n"))); - } - - public List<String> outputFromString(String output) { - return new ArrayList<>(Arrays.asList(output.split("\n"))); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java deleted file mode 100644 index 22d1a167a13..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class HardwareNodeComparatorTest { - - private HardwareInfo actualHardware; - private HardwareInfo nodeInfo; - - @Before - public void setup() { - actualHardware = new HardwareInfo(); - nodeInfo = new HardwareInfo(); - actualHardware.setMinCpuCores(24); - nodeInfo.setMinCpuCores(24); - actualHardware.setMinMainMemoryAvailableGb(16); - nodeInfo.setMinMainMemoryAvailableGb(16); - nodeInfo.setInterfaceSpeedMbs(10000); - actualHardware.setInterfaceSpeedMbs(10000); - actualHardware.setMinDiskAvailableGb(500); - nodeInfo.setMinDiskAvailableGb(500); - } - - @Test - public void compare_equal_hardware_should_create_emmpty_json() throws Exception { - String actualJson = new ObjectMapper().writeValueAsString(HardwareNodeComparator.compare(nodeInfo, actualHardware)); - String expectedJson = "{}"; - assertEquals(expectedJson, actualJson); - - } - - @Test - public void compare_different_amount_of_cores_should_create_json_with_actual_core_amount() throws Exception { - actualHardware.setMinCpuCores(4); - nodeInfo.setMinCpuCores(1); - String actualJson = new ObjectMapper().writeValueAsString(HardwareNodeComparator.compare(nodeInfo, actualHardware)); - String expectedJson = "{\"actualcpuCores\":4}"; - assertEquals(expectedJson, actualJson); - } - - @Test - public void compare_different_disk_type_should_create_json_with_actual_disk_type() throws Exception { - actualHardware.setDiskType(DiskType.SLOW); - nodeInfo.setDiskType(DiskType.FAST); - String actualJson = new ObjectMapper().writeValueAsString(HardwareNodeComparator.compare(nodeInfo, actualHardware)); - String expectedJson = "{\"actualDiskType\":\"SLOW\"}"; - assertEquals(expectedJson, actualJson); - } - - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java deleted file mode 100644 index 96af900095f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.yahoo.vespa.hosted.node.verification.Main; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class SpecVerifierTest { - - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources"; - private static final String CPU_INFO_PATH = RESOURCE_PATH + "/cpuinfoTest"; - private static final String MEMORY_INFO_PATH = RESOURCE_PATH + "/meminfoTest"; - private static final String FAST_DISK_TYPE_INFO_PATH = RESOURCE_PATH + "/DiskTypeFastDisk"; - private static final String DISK_SIZE_INFO_PATH = RESOURCE_PATH + "/filesize"; - private static final String NET_INTERFACE_INFO_PATH = RESOURCE_PATH + "/ifconfig"; - private static final String NET_INTERFACE_SPEED_INFO_PATH = RESOURCE_PATH + "/eth0"; - private static final String PING_RESPONSE = RESOURCE_PATH + "/validpingresponse"; - private final MockCommandExecutor commandExecutor = new MockCommandExecutor(); - - - @Test - public void spec_verification_with_failures() throws Exception { - commandExecutor.addCommand("cat " + CPU_INFO_PATH); - commandExecutor.addCommand("cat " + MEMORY_INFO_PATH); - commandExecutor.addCommand("cat " + FAST_DISK_TYPE_INFO_PATH); - commandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); - commandExecutor.addCommand("cat " + PING_RESPONSE); - - String result = Main.execute(new String[] { - "specification", - "-d", "2500", - "-m", "64", - "-c", "1.5", - "-s", "true", - "-b", "10000.0", - "-i", "10.11.12.13,::1234" - }, commandExecutor); - - assertEquals( - "{\"specVerificationReport\":{\"" + - "actualMemoryAvailable\":4.042128,\"" + - "actualDiskSpaceAvailable\":1760.0,\"" + - "actualInterfaceSpeed\":1000.0,\"" + - "actualcpuCores\":4,\"" + - "faultyIpAddresses\":[\"10.11.12.13\",\"0:0:0:0:0:0:0:1234\"]}}", result); - } - - @Test - public void benchmark_with_no_failures() throws Exception { - commandExecutor.addCommand("cat " + CPU_INFO_PATH); - commandExecutor.addCommand("cat " + MEMORY_INFO_PATH); - commandExecutor.addCommand("cat " + FAST_DISK_TYPE_INFO_PATH); - commandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); - commandExecutor.addCommand("cat " + PING_RESPONSE); - - String result = Main.execute(new String[] { - "specification", - "-d", "1760", - "-m", "4", - "-c", "4", - "-s", "true", - "-b", "1000" - }, commandExecutor); - - assertEquals("null", result); - } - - @Test - public void preserve_previous_benchmark_result() throws Exception { - commandExecutor.addCommand("cat " + CPU_INFO_PATH); - commandExecutor.addCommand("cat " + MEMORY_INFO_PATH); - commandExecutor.addCommand("cat " + FAST_DISK_TYPE_INFO_PATH); - commandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); - commandExecutor.addCommand("cat " + PING_RESPONSE); - - final String previousResult = "{\"specVerificationReport\":{\"actualMemoryAvailable\":4.042128}," + - "\"benchmarkReport\":{\"diskSpeedMbs\":49.0}}"; - - String result = Main.execute(new String[] { - "specification", - "-d", "1760", - "-m", "4", - "-c", "4", - "-s", "true", - "-b", "1000", - "-h", previousResult - }, commandExecutor); - - assertEquals("{\"benchmarkReport\":{\"diskSpeedMbs\":49.0}}", result); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk deleted file mode 100644 index 3b0eed77569..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk +++ /dev/null @@ -1,2 +0,0 @@ -Name Rota -sda 0
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest deleted file mode 100644 index 8e1b0236a6c..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest +++ /dev/null @@ -1,104 +0,0 @@ -processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 70 -model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz -stepping : 1 -cpu MHz : 2493.821 -cache size : 6144 KB -physical id : 0 -siblings : 1 -core id : 0 -cpu cores : 1 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat -bugs : -bogomips : 4987.64 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 70 -model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz -stepping : 1 -cpu MHz : 2493.821 -cache size : 6144 KB -physical id : 1 -siblings : 1 -core id : 0 -cpu cores : 1 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat -bugs : -bogomips : 5094.26 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 70 -model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz -stepping : 1 -cpu MHz : 2493.821 -cache size : 6144 KB -physical id : 2 -siblings : 1 -core id : 0 -cpu cores : 1 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat -bugs : -bogomips : 4390.19 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 70 -model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz -stepping : 1 -cpu MHz : 2493.821 -cache size : 6144 KB -physical id : 3 -siblings : 1 -core id : 0 -cpu cores : 1 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat -bugs : -bogomips : 5448.95 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 deleted file mode 100755 index 7b2d15a44c1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 +++ /dev/null @@ -1,15 +0,0 @@ - Supported ports: [ ] - Supported link modes: Not reported - Supported pause frame use: No - Supports auto-negotiation: No - Advertised link modes: Not reported - Advertised pause frame use: No - Advertised auto-negotiation: No - Speed: 1000Mb/s - Duplex: Full - Port: Twisted Pair - PHYAD: 0 - Transceiver: internal - Auto-negotiation: off - MDI-X: Unknown - Link detected: yes
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize deleted file mode 100644 index 535a509dcc1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize +++ /dev/null @@ -1,2 +0,0 @@ - PV Size 799.65 GB / not usable 0.00 GB - PV Size 960.35 GB / not usable 0.01 GB
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig deleted file mode 100755 index 51cd95e1351..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig +++ /dev/null @@ -1,9 +0,0 @@ -eth0 Link encap:Ethernet HWaddr 02:42:AC:11:00:03 - inet addr:172.17.0.3 Bcast:0.0.0.0 Mask:255.255.0.0 - inet6 addr: fe80::a00:27ff:fe70:e3f5/64 Scope:Link - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:52908 errors:0 dropped:0 overruns:0 frame:0 - TX packets:29527 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 - RX bytes:75003406 (71.5 MiB) TX bytes:1839045 (1.7 MiB) - diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 deleted file mode 100755 index 773d9576bde..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 +++ /dev/null @@ -1,7 +0,0 @@ -eth0 Link encap:Ethernet HWaddr 02:42:AC:11:00:03 - inet addr:172.17.0.3 Bcast:0.0.0.0 Mask:255.255.0.0 - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:52908 errors:0 dropped:0 overruns:0 frame:0 - TX packets:29527 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 - RX bytes:75003406 (71.5 MiB) TX bytes:1839045 (1.7 MiB)
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse deleted file mode 100644 index d111ac34ce1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse +++ /dev/null @@ -1 +0,0 @@ -this is no pingresponse ?!!? 34234
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest deleted file mode 100644 index df2282d9528..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest +++ /dev/null @@ -1,45 +0,0 @@ -MemTotal: 4042128 kB -MemFree: 3584108 kB -MemAvailable: 3562648 kB -Buffers: 9568 kB -Cached: 315836 kB -SwapCached: 0 kB -Active: 176620 kB -Inactive: 202120 kB -Active(anon): 113320 kB -Inactive(anon): 105560 kB -Active(file): 63300 kB -Inactive(file): 96560 kB -Unevictable: 0 kB -Mlocked: 0 kB -SwapTotal: 1048572 kB -SwapFree: 1048572 kB -Dirty: 0 kB -Writeback: 0 kB -AnonPages: 53348 kB -Mapped: 49140 kB -Shmem: 165548 kB -Slab: 54504 kB -SReclaimable: 38372 kB -SUnreclaim: 16132 kB -KernelStack: 3552 kB -PageTables: 1368 kB -NFS_Unstable: 0 kB -Bounce: 0 kB -WritebackTmp: 0 kB -CommitLimit: 3069636 kB -Committed_AS: 810504 kB -VmallocTotal: 34359738367 kB -VmallocUsed: 0 kB -VmallocChunk: 0 kB -AnonHugePages: 0 kB -ShmemHugePages: 0 kB -ShmemPmdMapped: 0 kB -HugePages_Total: 0 -HugePages_Free: 0 -HugePages_Rsvd: 0 -HugePages_Surp: 0 -Hugepagesize: 2048 kB -DirectMap4k: 24576 kB -DirectMap2M: 2072576 kB -DirectMap1G: 4194304 kB diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse deleted file mode 100644 index c227083464f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse +++ /dev/null @@ -1 +0,0 @@ -0
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java deleted file mode 100644 index 0cf6bb0907b..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class CPURetrieverTest { - - private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest"; - private HardwareInfo hardwareInfo; - private MockCommandExecutor commandExecutor; - private CPURetriever cpu; - private static final double DELTA = 0.1; - - @Before - public void setup() { - hardwareInfo = new HardwareInfo(); - commandExecutor = new MockCommandExecutor(); - cpu = new CPURetriever(hardwareInfo, commandExecutor); - } - - @Test - public void updateInfo_should_write_numOfCpuCores_to_hardware_info() { - commandExecutor.addCommand("cat " + FILENAME); - cpu.updateInfo(); - double expectedAmountOfCores = 4; - assertEquals(expectedAmountOfCores, hardwareInfo.getMinCpuCores(), DELTA); - } - - @Test - public void parseCPUInfoFile_should_return_valid_ArrayList() throws IOException { - List<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); - List<ParseResult> ParseResults = cpu.parseCPUInfoFile(commandOutput); - String expectedSearchWord = "cpu MHz"; - String expectedValue = "2493.821"; - - assertEquals(expectedSearchWord, ParseResults.get(0).getSearchWord()); - assertEquals(expectedValue, ParseResults.get(0).getValue()); - - assertEquals(expectedSearchWord, ParseResults.get(1).getSearchWord()); - assertEquals(expectedValue, ParseResults.get(1).getValue()); - - assertEquals(expectedSearchWord, ParseResults.get(2).getSearchWord()); - assertEquals(expectedValue, ParseResults.get(2).getValue()); - - assertEquals(expectedSearchWord, ParseResults.get(3).getSearchWord()); - assertEquals(expectedValue, ParseResults.get(3).getValue()); - } - - @Test - public void setCpuCores_counts_cores_correctly() { - List<ParseResult> parseResults = new ArrayList<>(); - parseResults.add(new ParseResult("cpu MHz", "2000")); - parseResults.add(new ParseResult("cpu MHz", "2000")); - parseResults.add(new ParseResult("cpu MHz", "2000")); - cpu.setCpuCores(parseResults); - int expectedCpuCores = 3; - assertEquals(expectedCpuCores, hardwareInfo.getMinCpuCores()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java deleted file mode 100644 index a72cca1c72a..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author sgrostad - * @author olaaun - */ - -public class DiskRetrieverTest { - - private MockCommandExecutor commandExecutor; - private HardwareInfo hardwareInfo; - private DiskRetriever diskRetriever; - private static String CAT_RESOURCE_PATH = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; - private static final double DELTA = 0.1; - - @Before - public void setup() { - hardwareInfo = new HardwareInfo(); - commandExecutor = new MockCommandExecutor(); - diskRetriever = new DiskRetriever(hardwareInfo, commandExecutor); - } - - @Test - public void updateInfo_should_store_diskType_and_diskSize_in_hardware_info() { - commandExecutor.addCommand(CAT_RESOURCE_PATH + "DiskTypeFastDisk"); - commandExecutor.addCommand(CAT_RESOURCE_PATH + "filesize"); - diskRetriever.updateInfo(); - assertEquals(DiskType.FAST, hardwareInfo.getDiskType()); - double expectedSize = 1760.0; - assertEquals(expectedSize, hardwareInfo.getMinDiskAvailableGb(), DELTA); - } - - @Test - public void updateDiskType__should_store_diskType_in_hardwareInfo() { - commandExecutor.addCommand(CAT_RESOURCE_PATH + "DiskTypeFastDisk"); - diskRetriever.updateDiskType(); - assertEquals(DiskType.FAST, hardwareInfo.getDiskType()); - } - - @Test - public void updateDiskSize__should_store_diskSize_in_hardwareInfo() { - commandExecutor.addCommand(CAT_RESOURCE_PATH + "filesize"); - diskRetriever.updateDiskSize(); - double expectedSize = 1760.0; - assertEquals(expectedSize, hardwareInfo.getMinDiskAvailableGb(), DELTA); - } - - @Test - public void parseDiskType_should_find_fast_disk() throws Exception { - diskRetriever = new DiskRetriever(hardwareInfo, commandExecutor); - List<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 0"); - ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); - ParseResult expectedParseResult = new ParseResult("sda", "0"); - assertEquals(expectedParseResult, parseResult); - } - - @Test - public void parseDiskType_should_not_find_fast_disk() throws Exception { - List<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 1"); - ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); - ParseResult expectedParseResult = new ParseResult("sda", "1"); - assertEquals(expectedParseResult, parseResult); - } - - @Test - public void parseDiskType_with_invalid_outputstream_does_not_contain_searchword_should_throw_exception() { - List<String> mockOutput = commandExecutor.outputFromString("Name Rota"); - try { - diskRetriever.parseDiskType(mockOutput); - fail("Should have thrown IOException when outputstream doesn't contain search word"); - } catch (IOException e) { - String expectedExceptionMessage = "Parsing for disk type failed"; - assertEquals(expectedExceptionMessage, e.getMessage()); - } - - } - - @Test - public void parseDiskSize_should_find_size_from_file_and_insert_into_parseResult() throws Exception { - String filepath = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize"; - List<String> mockOutput = MockCommandExecutor.readFromFile(filepath); - List<ParseResult> parseResults = diskRetriever.parseDiskSize(mockOutput); - ParseResult expectedParseResult1 = new ParseResult("Size", "799.65"); - assertEquals(expectedParseResult1, parseResults.get(0)); - ParseResult expectedParseResult2 = new ParseResult("Size", "960.35"); - assertEquals(expectedParseResult2, parseResults.get(1)); - } - - @Test - public void setDiskType_invalid_ParseResult_should_set_fastDisk_to_invalid() { - ParseResult parseResult = new ParseResult("Invalid", "Invalid"); - diskRetriever.setDiskType(parseResult); - HardwareInfo.DiskType expectedDiskType = HardwareInfo.DiskType.UNKNOWN; - assertEquals(expectedDiskType, hardwareInfo.getDiskType()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java deleted file mode 100644 index 0ebf1a2b26a..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - - -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - -/** - * @author sgrostad - * @author olaaun - */ - -public class HardwareInfoRetrieverTest { - - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; - private static final String CPU_INFO_PATH = RESOURCE_PATH + "cpuinfoTest"; - private static final String MEMORY_INFO_PATH = RESOURCE_PATH + "meminfoTest"; - private static final String DISK_TYPE_INFO_PATH = RESOURCE_PATH + "DiskTypeFastDisk"; - private static final String DISK_SIZE_INFO_PATH = RESOURCE_PATH + "filesize"; - private static final String NET_INTERFACE_INFO_PATH = RESOURCE_PATH + "ifconfigNoIpv6"; - private static final String NET_INTERFACE_SPEED_INFO_PATH = RESOURCE_PATH + "eth0"; - private static String PING_RESPONSE = RESOURCE_PATH + "invalidpingresponse"; - private MockCommandExecutor mockCommandExecutor; - private HardwareInfo expectedHardwareInfo; - private VerifierSettings verifierSettings = spy(new VerifierSettings()); - private static final double DELTA = 0.1; - - @Before - public void setup() { - mockCommandExecutor = new MockCommandExecutor(); - mockCommandExecutor.addCommand("cat " + CPU_INFO_PATH); - mockCommandExecutor.addCommand("cat " + MEMORY_INFO_PATH); - mockCommandExecutor.addCommand("cat " + DISK_TYPE_INFO_PATH); - mockCommandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); - mockCommandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); - mockCommandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); - mockCommandExecutor.addCommand("cat " + PING_RESPONSE); - - expectedHardwareInfo = new HardwareInfo(); - expectedHardwareInfo.setMinCpuCores(4); - expectedHardwareInfo.setMinMainMemoryAvailableGb(4.042128); - expectedHardwareInfo.setInterfaceSpeedMbs(1000); - expectedHardwareInfo.setMinDiskAvailableGb(1760.0); - expectedHardwareInfo.setIpv4Interface(true); - expectedHardwareInfo.setIpv6Interface(false); - expectedHardwareInfo.setIpv6Connection(false); - expectedHardwareInfo.setDiskType(HardwareInfo.DiskType.FAST); - } - - @Test - public void retriever_should_return_valid_HardwareInfo() { - doReturn(true).when(verifierSettings).isCheckIPv6(); - HardwareInfo actualHardwareInfo = HardwareInfoRetriever.retrieve(mockCommandExecutor, verifierSettings); - assertEquals(expectedHardwareInfo.getMinDiskAvailableGb(), actualHardwareInfo.getMinDiskAvailableGb(), DELTA); - assertEquals(expectedHardwareInfo.getMinMainMemoryAvailableGb(), actualHardwareInfo.getMinMainMemoryAvailableGb(), DELTA); - assertEquals(expectedHardwareInfo.getMinCpuCores(), actualHardwareInfo.getMinCpuCores()); - assertEquals(expectedHardwareInfo.getIpv4Interface(), actualHardwareInfo.getIpv4Interface()); - assertEquals(expectedHardwareInfo.getIpv6Interface(), actualHardwareInfo.getIpv6Interface()); - assertEquals(expectedHardwareInfo.getInterfaceSpeedMbs(), actualHardwareInfo.getInterfaceSpeedMbs(), DELTA); - assertEquals(expectedHardwareInfo.getDiskType(), actualHardwareInfo.getDiskType()); - assertEquals(expectedHardwareInfo.isIpv6Connection(), actualHardwareInfo.isIpv6Connection()); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java deleted file mode 100644 index 1722abdc7b8..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class MemoryRetrieverTest { - - private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest"; - private HardwareInfo hardwareInfo; - private MockCommandExecutor commandExecutor; - private MemoryRetriever memory; - private final double DELTA = 0.1; - - @Before - public void setup() { - hardwareInfo = new HardwareInfo(); - commandExecutor = new MockCommandExecutor(); - memory = new MemoryRetriever(hardwareInfo, commandExecutor); - } - - @Test - public void updateInfo_should_set_memory_available_in_hardwareInfo() { - commandExecutor.addCommand("cat " + FILENAME); - memory.updateInfo(); - double expectedMemory = 4.042128; - assertEquals(expectedMemory, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); - } - - @Test - public void parseMemInfoFile_should_return_valid_parseResult() throws IOException { - List<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); - ParseResult parseResult = memory.parseMemInfoFile(commandOutput); - ParseResult expectedParseResult = new ParseResult("MemTotal", "4042128 kB"); - assertEquals(expectedParseResult, parseResult); - } - - @Test - public void updateMemoryInfo_valid_input() { - ParseResult testParseResult = new ParseResult("MemTotal", "4042128"); - memory.updateMemoryInfo(testParseResult); - double expectedMemory = 4.042128; - assertEquals(expectedMemory, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); - } - - @Test - public void convertToGB_valid_input() { - String testTotMem = "4042128"; - double expectedTotMem = 4.042128; - assertEquals(expectedTotMem, memory.convertKBToGB(testTotMem), DELTA); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java deleted file mode 100644 index dad73a9bb90..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - -/** - * @author sgrostad - * @author olaaun - */ - -public class NetRetrieverTest { - - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; - private static final String NET_FIND_INTERFACE = RESOURCE_PATH + "ifconfig"; - private static final String NET_CHECK_INTERFACE_SPEED = RESOURCE_PATH + "eth0"; - private static String VALID_PING_RESPONSE = RESOURCE_PATH + "validpingresponse"; - private static String INVALID_PING_RESPONSE = RESOURCE_PATH + "invalidpingresponse"; - private static String PING_SEARCH_WORD = "\\d+\\.?\\d*"; - private HardwareInfo hardwareInfo; - private MockCommandExecutor commandExecutor; - private NetRetriever net; - private List<ParseResult> parseResults; - private VerifierSettings verifierSettings = spy(new VerifierSettings()); - private static final double DELTA = 0.1; - - @Before - public void setup() { - hardwareInfo = new HardwareInfo(); - commandExecutor = new MockCommandExecutor(); - doReturn(true).when(verifierSettings).isCheckIPv6(); - net = new NetRetriever(hardwareInfo, commandExecutor, verifierSettings); - parseResults = new ArrayList<>(); - } - - @Test - public void updateInfo_should_store_ipv4_ipv6_interface_and_interface_speed() { - commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); - commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED); - commandExecutor.addCommand("cat " + VALID_PING_RESPONSE); - net.updateInfo(); - assertTrue(hardwareInfo.getIpv4Interface()); - assertTrue(hardwareInfo.getIpv6Interface()); - assertTrue(hardwareInfo.isIpv6Connection()); - double expectedInterfaceSpeed = 1000; - assertEquals(expectedInterfaceSpeed, hardwareInfo.getInterfaceSpeedMbs(), DELTA); - } - - @Test - public void findInterfaceSpeed_valid_input() { - commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); - commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED); - parseResults = net.findInterface(); - net.findInterfaceSpeed(parseResults); - ParseResult expectedParseResults = new ParseResult("Speed", "1000Mb/s"); - assertEquals(expectedParseResults, parseResults.get(2)); - } - - @Test - public void parseNetInterface_get_ipv_from_ifconfig_testFile() throws IOException { - List<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE); - parseResults = net.parseNetInterface(mockOutput); - net.updateHardwareInfoWithNet(parseResults); - assertTrue(hardwareInfo.getIpv4Interface()); - assertTrue(hardwareInfo.getIpv6Interface()); - } - - @Test - public void parseNetInterface_get_ipv_from_ifconfigNotIpv6_testFile() throws IOException { - List<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE + "NoIpv6"); - parseResults = net.parseNetInterface(mockOutput); - List<ParseResult> expextedParseResults = Collections.singletonList(new ParseResult("inet", "inet")); - assertEquals(expextedParseResults, parseResults); - } - - @Test - public void parseInterfaceSpeed_get_interfaceSpeed_from_eth0_testFile() throws IOException { - List<String> mockOutput = MockCommandExecutor.readFromFile("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0"); - ParseResult parseResult = net.parseInterfaceSpeed(mockOutput); - ParseResult expectedParseResult = new ParseResult("Speed", "1000Mb/s"); - assertEquals(expectedParseResult, parseResult); - } - - - @Test - public void updateHardwareinfoWithNet_valid_input() { - parseResults.add(new ParseResult("inet", "inet")); - parseResults.add(new ParseResult("inet6", "inet6")); - parseResults.add(new ParseResult("Speed", "1000Mb/s")); - net.updateHardwareInfoWithNet(parseResults); - double expectedInterfaceSpeed = 1000; - assertEquals(expectedInterfaceSpeed, hardwareInfo.getInterfaceSpeedMbs(), DELTA); - assertTrue(hardwareInfo.getIpv4Interface()); - assertTrue(hardwareInfo.getIpv6Interface()); - } - - @Test - public void stripInterfaceSpeed_should_return_correct_double() { - String interfaceSpeedToConvert = "1000Mb/s"; - double expectedInterfaceSpeed = 1000; - double actualInterfaceSpeed = net.convertInterfaceSpeed(interfaceSpeedToConvert); - assertEquals(expectedInterfaceSpeed, actualInterfaceSpeed, DELTA); - } - - @Test - public void parsePingResponse_valid_ping_response_should_return_ipv6_connectivity() throws IOException { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_PING_RESPONSE); - ParseResult parseResult = net.parsePingResponse(mockCommandOutput); - String expectedPing = "0"; - assertEquals(expectedPing, parseResult.getValue()); - } - - @Test - public void parsePingResponse_invalid_ping_response_should_throw_IOException() throws IOException { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_PING_RESPONSE); - try { - net.parsePingResponse(mockCommandOutput); - fail("Expected an IOException to be thrown"); - } catch (IOException e) { - String expectedExceptionMessage = "Failed to parse ping output."; - assertEquals(expectedExceptionMessage, e.getMessage()); - } - } - - @Test - public void setIpv6Connectivity_valid_ping_response_should_return_ipv6_connectivity() { - ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "0"); - net.setIpv6Connectivity(parseResult); - assertTrue(hardwareInfo.isIpv6Connection()); - } - - @Test - public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_1() { - ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "100"); - net.setIpv6Connectivity(parseResult); - assertFalse(hardwareInfo.isIpv6Connection()); - } - - @Test - public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_2() { - ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "invalid"); - net.setIpv6Connectivity(parseResult); - assertFalse(hardwareInfo.isIpv6Connection()); - } - -} @@ -100,7 +100,6 @@ <module>model-integration</module> <module>node-repository</module> <module>node-admin</module> - <module>node-maintainer</module> <module>orchestrator-restapi</module> <module>orchestrator</module> <module>parent</module> |